DL_Project_vFinal.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "L4",
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "02494cb422e04488a09c24f967cce71c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5d78a245a86642488fe8781e20324f76",
              "IPY_MODEL_14eada7fbdbe4b649a3e0c6e1132efbb",
              "IPY_MODEL_d0cf66d00a9e49a0963c67ad435f9776"
            ],
            "layout": "IPY_MODEL_4171bba2cd3749dbae50257b114c77e1"
          }
        },
        "5d78a245a86642488fe8781e20324f76": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8748d002f0ba49f182fa3042e1140915",
            "placeholder": "​",
            "style": "IPY_MODEL_5169c933246a4710bf715a6c6b8a4b26",
            "value": "model.safetensors: 100%"
          }
        },
        "14eada7fbdbe4b649a3e0c6e1132efbb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f69d7d2ddf384e628598dabac90a6e0f",
            "max": 548105171,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_da2d8ef7a418414c847a355d2b47b850",
            "value": 548105171
          }
        },
        "d0cf66d00a9e49a0963c67ad435f9776": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6a4c2c6ecf18408e9dc240650e6c53fa",
            "placeholder": "​",
            "style": "IPY_MODEL_388f33852aac4e5c88ebee7c9d5e7f29",
            "value": " 548M/548M [00:02&lt;00:00, 276MB/s]"
          }
        },
        "4171bba2cd3749dbae50257b114c77e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8748d002f0ba49f182fa3042e1140915": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5169c933246a4710bf715a6c6b8a4b26": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f69d7d2ddf384e628598dabac90a6e0f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "da2d8ef7a418414c847a355d2b47b850": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6a4c2c6ecf18408e9dc240650e6c53fa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "388f33852aac4e5c88ebee7c9d5e7f29": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d0ce301793ef4f44b77864d1aa9e4097": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3754563b52a049439c9620114d5e3f8f",
              "IPY_MODEL_a420516444164ff4a631c464e1b83c5b",
              "IPY_MODEL_4d1d6263cd064efcbd56737cecd63ebf"
            ],
            "layout": "IPY_MODEL_a11ac25efba94e2586dd1f4ef0cba50a"
          }
        },
        "3754563b52a049439c9620114d5e3f8f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7801c37b687149bba28172a12658476e",
            "placeholder": "​",
            "style": "IPY_MODEL_da2be2ad477e4b96b1792c174d801f97",
            "value": "generation_config.json: 100%"
          }
        },
        "a420516444164ff4a631c464e1b83c5b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_71f0299c28d84c5d9a2f9c65a2423fd9",
            "max": 124,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f5d27337a88849658f6f583f2dabd53e",
            "value": 124
          }
        },
        "4d1d6263cd064efcbd56737cecd63ebf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6aac42473a5143b484483082e0590956",
            "placeholder": "​",
            "style": "IPY_MODEL_afedae4055044db6b42e7697b732b7cb",
            "value": " 124/124 [00:00&lt;00:00, 11.3kB/s]"
          }
        },
        "a11ac25efba94e2586dd1f4ef0cba50a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7801c37b687149bba28172a12658476e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "da2be2ad477e4b96b1792c174d801f97": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "71f0299c28d84c5d9a2f9c65a2423fd9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f5d27337a88849658f6f583f2dabd53e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6aac42473a5143b484483082e0590956": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "afedae4055044db6b42e7697b732b7cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/noahminds/fake_amazon_review_discriminator/blob/main/Final_Project_v1_LastHiddenLayer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q2A3aFf724em",
        "outputId": "302750f2-507e-4f7f-b5fa-62c831130997"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive', force_remount=True)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset_path = \"/content/drive/MyDrive/CS497/Final Project/Fake reviews data files/generated-datasets/\"\n",
        "model_weights_path = \"/content/drive/MyDrive/CS497/Final Project/Fake reviews data files/gpt-2models/models/\""
      ],
      "metadata": {
        "id": "_Uu_dpp_4B3t"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GYOAM7Stz7id"
      },
      "outputs": [],
      "source": [
        "!pip install -q transformers"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn import metrics\n",
        "import transformers\n",
        "import torch\n",
        "import json\n",
        "from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler, Subset\n",
        "from transformers import AutoTokenizer, GPT2TokenizerFast, TFGPT2Model, GPT2LMHeadModel, GPT2Config\n",
        "import torch.nn.functional as F\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split"
      ],
      "metadata": {
        "id": "njWfuypS1J0B"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from torch import cuda\n",
        "device = 'cuda' if cuda.is_available() else 'cpu'"
      ],
      "metadata": {
        "id": "wIDW0Xns1MK4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "file_name = dataset_path + \"Generated_Sentences_Books_5.csv\"\n",
        "df = pd.read_csv(file_name)\n",
        "\n",
        "print(df[df['label'] == 'CG'].head())\n",
        "print(df[df['label'] == 'OR'].head())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wVsOgM8yhc8x",
        "outputId": "1c280254-ad5b-4d1a-fbfd-00892ead9d18"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  category  rating label                                              text_\n",
            "0  Books_5     5.0    CG  I'm hooked on this writer and will be reading ...\n",
            "1  Books_5     5.0    CG  Good book and exactly as described. The charac...\n",
            "2  Books_5     5.0    CG  Fitting that the ultimate survivor of an epide...\n",
            "3  Books_5     5.0    CG  Absolutely loved every word!  We have the book...\n",
            "4  Books_5     5.0    CG  I loved it! It was a real, believable, and eng...\n",
            "   category  rating label                                              text_\n",
            "24  Books_5     4.0    OR  It's witty and intriguing. I hang on to every ...\n",
            "25  Books_5     5.0    OR  Good read. I really enjoyed the characters in ...\n",
            "26  Books_5     5.0    OR  It's Dr. Seuss, how can you miss??? Great qual...\n",
            "27  Books_5     5.0    OR  Great read.  Makes you wish you lived in Nantu...\n",
            "28  Books_5     4.0    OR  A good read which keeps you wondering until th...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Splitting into training, validation, and testing sets"
      ],
      "metadata": {
        "id": "E_EhnlJbluX9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train_df, temp_df = train_test_split(df, test_size=0.1, stratify=df['label'], random_state=42)\n",
        "valid_df, test_df = train_test_split(temp_df, test_size=0.5, stratify=temp_df['label'], random_state=42)\n",
        "\n",
        "print(f\"FULL Dataset: {len(df)}\")\n",
        "print(f\"TRAIN Dataset Size: {len(train_df)}\")\n",
        "print(f\"VALID Dataset Size: {len(valid_df)}\")\n",
        "\n",
        "print(\"\\nTraining set:\")\n",
        "print(train_df['label'].value_counts(normalize=True))\n",
        "print(train_df.head(2))\n",
        "\n",
        "print(\"\\nValidation set:\")\n",
        "print(valid_df['label'].value_counts(normalize=True))\n",
        "print(valid_df.head(2))\n",
        "\n",
        "print(\"\\nTest set:\")\n",
        "print(test_df['label'].value_counts(normalize=True))\n",
        "print(test_df.head(2))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LljME-qTlte2",
        "outputId": "d98541c3-314e-4e54-a0fa-cc887be3301b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "FULL Dataset: 4370\n",
            "TRAIN Dataset Size: 3933\n",
            "VALID Dataset Size: 218\n",
            "\n",
            "Training set:\n",
            "label\n",
            "OR    0.500127\n",
            "CG    0.499873\n",
            "Name: proportion, dtype: float64\n",
            "     category  rating label                                              text_\n",
            "1264  Books_5     5.0    OR  I love when King really gets inside the head o...\n",
            "3446  Books_5     4.0    CG  This is a really moving book. The characters a...\n",
            "\n",
            "Validation set:\n",
            "label\n",
            "CG    0.5\n",
            "OR    0.5\n",
            "Name: proportion, dtype: float64\n",
            "     category  rating label                                              text_\n",
            "3796  Books_5     1.0    CG  Unfortunately this book falls far short of the...\n",
            "3065  Books_5     5.0    CG  Carl Sagan is an astounding writer. The story ...\n",
            "\n",
            "Test set:\n",
            "label\n",
            "CG    0.502283\n",
            "OR    0.497717\n",
            "Name: proportion, dtype: float64\n",
            "     category  rating label                                              text_\n",
            "2775  Books_5     4.0    CG  I really liked the overall story. The characte...\n",
            "643   Books_5     5.0    CG  great book  I still find it entertaining.  I'v...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Instantiating and testing the tokenizer"
      ],
      "metadata": {
        "id": "a0rkJVIS7ul5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# We get the max and mean length of the text to set an appropriate\n",
        "# max_length parameter for tokenization\n",
        "max_length = df['text_'].apply(len).max()\n",
        "average_length = df['text_'].apply(len).mean()\n",
        "\n",
        "print(f\"Maximum length of text: {max_length}\")\n",
        "print(f\"Average length of text: {average_length}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6XxmM6i9ACNH",
        "outputId": "b754016b-4070-4d0f-d57d-67b43cfacec0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Maximum length of text: 2114\n",
            "Average length of text: 432.34279176201375\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Choose a maximum input sequence length of 512\n",
        "MAX_LEN = 512\n",
        "\n",
        "# Initialize tokenizer\n",
        "tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')\n",
        "tokenizer.pad_token = tokenizer.eos_token\n",
        "\n",
        "# Assuming labels are stored as 'OR' and 'CG' in the 'label' column\n",
        "labels_dict = {'OR': 0, 'CG': 1}\n",
        "\n",
        "class ReviewDataset(Dataset):\n",
        "    def __init__(self, dataframe):\n",
        "        self.dataframe = dataframe\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.dataframe)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        return {\n",
        "            'text_': self.dataframe.iloc[idx]['text_'],\n",
        "            'label': self.dataframe.iloc[idx]['label']\n",
        "        }\n",
        "\n",
        "# Custom collate function to handle tokenization and batching\n",
        "def collate_fn(batch):\n",
        "    texts = [item['text_'] for item in batch]\n",
        "    class_labels = [labels_dict[item['label']] for item in batch]  # Convert labels to indices\n",
        "\n",
        "    # Calculate lengths before padding for use in the model\n",
        "    lengths = [len(tokenizer.encode(text)) for text in texts]\n",
        "\n",
        "    encodings = tokenizer(texts,\n",
        "                          return_tensors='pt',\n",
        "                          max_length=MAX_LEN,\n",
        "                          padding='max_length',\n",
        "                          truncation=True)\n",
        "\n",
        "    # Add labels to the dictionary\n",
        "    encodings['labels'] = torch.tensor(class_labels, dtype=torch.long)\n",
        "    encodings['lengths'] = torch.tensor(lengths)\n",
        "\n",
        "    return encodings\n",
        "\n",
        "# Creating the dataset and data loader\n",
        "example_df = pd.DataFrame({\n",
        "    'category': ['Books_5', 'Books_5'],\n",
        "    'rating': [4.0, 5.0],\n",
        "    'label': ['CG', 'OR'],\n",
        "    'text_': ['I really liked the overall story...', 'great book I still find it entertaining...']\n",
        "})\n",
        "\n",
        "dataset = ReviewDataset(example_df)\n",
        "example_data_loader = DataLoader(dataset, batch_size=2, collate_fn=collate_fn)\n",
        "\n",
        "# Iterate over DataLoader\n",
        "for batch in example_data_loader:\n",
        "    decoded_texts = [tokenizer.decode(ids, skip_special_tokens=True) for ids in batch['input_ids']]\n",
        "    print(decoded_texts)\n",
        "    print(batch['input_ids'])\n",
        "    print(batch['attention_mask'])  # Attention masks\n",
        "    print(batch['lengths']) # Length of original input sequence without appended padding tokens\n",
        "    print(batch['labels'])  # Labels\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xLco6s-CvSYB",
        "outputId": "c5d42bfd-fb70-4762-974e-8f6949b09e90"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['I really liked the overall story...', 'great book I still find it entertaining...']\n",
            "tensor([[   40,  1107,  8288,  ..., 50256, 50256, 50256],\n",
            "        [18223,  1492,   314,  ..., 50256, 50256, 50256]])\n",
            "tensor([[1, 1, 1,  ..., 0, 0, 0],\n",
            "        [1, 1, 1,  ..., 0, 0, 0]])\n",
            "tensor([7, 8])\n",
            "tensor([1, 0])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Tokenizing the datasets and creating dataloaders"
      ],
      "metadata": {
        "id": "xBqWouRS76LM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "BATCHSIZE = 32\n",
        "\n",
        "train_dataset = ReviewDataset(train_df)\n",
        "valid_dataset = ReviewDataset(valid_df)\n",
        "test_dataset = ReviewDataset(test_df)\n",
        "\n",
        "exclude_batches = [25, 41]  # Zero-indexed; hence, 26-1 and 42-1\n",
        "exclude_indices = []\n",
        "for batch_num in exclude_batches:\n",
        "    start_index = batch_num * BATCHSIZE\n",
        "    end_index = start_index + BATCHSIZE\n",
        "    exclude_indices.extend(range(start_index, end_index))\n",
        "\n",
        "# Calculate the indices to include\n",
        "all_indices = set(range(len(train_dataset)))\n",
        "include_indices = list(all_indices - set(exclude_indices))\n",
        "\n",
        "# Create a new subset of data excluding the specified batches\n",
        "filtered_train_dataset = Subset(train_dataset, include_indices)\n",
        "\n",
        "train_dataloader = DataLoader(filtered_train_dataset, batch_size=BATCHSIZE, collate_fn=collate_fn, shuffle=True)\n",
        "valid_dataloader = DataLoader(valid_dataset, batch_size=BATCHSIZE, collate_fn=collate_fn, shuffle=True)\n",
        "test_dataloader = DataLoader(test_dataset, batch_size=BATCHSIZE, collate_fn=collate_fn, shuffle=True)"
      ],
      "metadata": {
        "id": "sl35B1Os6Ea3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Initializing the researchers' fine-tuned model for Book reviews"
      ],
      "metadata": {
        "id": "VpnDJEo3se5F"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Initialize the model with the GPT-2 configuration\n",
        "model = GPT2LMHeadModel.from_pretrained('gpt2')\n",
        "\n",
        "# Retrieve the fine-tuned model checkpoint\n",
        "model_path = model_weights_path + \"gpt2-finetuned-amazon-reviews-sample-1cycle-Books_5.pth\"\n",
        "checkpoint = torch.load(model_path, map_location=device)\n",
        "\n",
        "# Access the model state_dict\n",
        "model_state_dict = checkpoint['model']\n",
        "\n",
        "# Load the state dictionary\n",
        "model.load_state_dict(model_state_dict, strict = False)\n",
        "model.to(device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 550,
          "referenced_widgets": [
            "02494cb422e04488a09c24f967cce71c",
            "5d78a245a86642488fe8781e20324f76",
            "14eada7fbdbe4b649a3e0c6e1132efbb",
            "d0cf66d00a9e49a0963c67ad435f9776",
            "4171bba2cd3749dbae50257b114c77e1",
            "8748d002f0ba49f182fa3042e1140915",
            "5169c933246a4710bf715a6c6b8a4b26",
            "f69d7d2ddf384e628598dabac90a6e0f",
            "da2d8ef7a418414c847a355d2b47b850",
            "6a4c2c6ecf18408e9dc240650e6c53fa",
            "388f33852aac4e5c88ebee7c9d5e7f29",
            "d0ce301793ef4f44b77864d1aa9e4097",
            "3754563b52a049439c9620114d5e3f8f",
            "a420516444164ff4a631c464e1b83c5b",
            "4d1d6263cd064efcbd56737cecd63ebf",
            "a11ac25efba94e2586dd1f4ef0cba50a",
            "7801c37b687149bba28172a12658476e",
            "da2be2ad477e4b96b1792c174d801f97",
            "71f0299c28d84c5d9a2f9c65a2423fd9",
            "f5d27337a88849658f6f583f2dabd53e",
            "6aac42473a5143b484483082e0590956",
            "afedae4055044db6b42e7697b732b7cb"
          ]
        },
        "id": "MHYSqzF5rW9g",
        "outputId": "ffdbfdfe-e871-4177-b6a4-8b9569e37936"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/548M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "02494cb422e04488a09c24f967cce71c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "generation_config.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d0ce301793ef4f44b77864d1aa9e4097"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GPT2LMHeadModel(\n",
              "  (transformer): GPT2Model(\n",
              "    (wte): Embedding(50257, 768)\n",
              "    (wpe): Embedding(1024, 768)\n",
              "    (drop): Dropout(p=0.1, inplace=False)\n",
              "    (h): ModuleList(\n",
              "      (0-11): 12 x GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (act): NewGELUActivation()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "  )\n",
              "  (lm_head): Linear(in_features=768, out_features=50257, bias=False)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Adding a head to the model to retask the fine-tuned gpt-2 generator model for discrimination"
      ],
      "metadata": {
        "id": "7Yv9lnUyshe5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class GPT2Discriminator(torch.nn.Module):\n",
        "    def __init__(self, gpt2_model):\n",
        "        super(GPT2Discriminator, self).__init__()\n",
        "        self.l1 = gpt2_model  # Pre-trained GPT2LMHeadModel from Hugging Face with weights loaded from researchers fine-tuned state dictionary\n",
        "        self.dropout = torch.nn.Dropout(0.1)  # Dropout layer\n",
        "        self.ln = torch.nn.LayerNorm(model.config.n_embd)  # Layer normalization\n",
        "        self.l3 = torch.nn.Linear(model.config.n_embd, 2)  # Linear layer (tensor: [768, 2]) maps output to our two classes\n",
        "\n",
        "    def forward(self, input_ids, attention_mask, seq_lengths):\n",
        "        outputs = self.l1(input_ids=input_ids, attention_mask=attention_mask, output_hidden_states=True)\n",
        "\n",
        "        # We extract representations of the last non-padding token in our input sequence from the final hidden state of the model\n",
        "        last_hidden_state = outputs.hidden_states[-1]  # tensor: [batch_size, seq_len=512, hidden_size=768]\n",
        "\n",
        "        # Apply dropout and layer normalization\n",
        "        last_hidden_state = self.dropout(last_hidden_state)\n",
        "        last_hidden_state = self.ln(last_hidden_state)\n",
        "\n",
        "        batch_size = last_hidden_state.size(0)  # typically 64\n",
        "        seq_indices = torch.arange(batch_size, device=input_ids.device)  # tensor: [batch_size]\n",
        "        last_token_indices = seq_lengths - 1  # tensor: [batch_size]\n",
        "        cls_output = last_hidden_state[seq_indices, last_token_indices]  # Shape: [batch_size, hidden_size=768]\n",
        "\n",
        "        # Compute the logits using the linear layer\n",
        "        logits = self.l3(cls_output)  # Shape: [batch_size, 2]\n",
        "\n",
        "        return logits\n",
        "\n",
        "\n",
        "model = GPT2Discriminator(model)\n",
        "model.to(device)"
      ],
      "metadata": {
        "id": "qpZx0m3IghD2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fa0c49e1-dcbf-4a56-acf5-465ee8c5fef8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GPT2Discriminator(\n",
              "  (l1): GPT2LMHeadModel(\n",
              "    (transformer): GPT2Model(\n",
              "      (wte): Embedding(50257, 768)\n",
              "      (wpe): Embedding(1024, 768)\n",
              "      (drop): Dropout(p=0.1, inplace=False)\n",
              "      (h): ModuleList(\n",
              "        (0-11): 12 x GPT2Block(\n",
              "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "          (attn): GPT2Attention(\n",
              "            (c_attn): Conv1D()\n",
              "            (c_proj): Conv1D()\n",
              "            (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "            (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "          (mlp): GPT2MLP(\n",
              "            (c_fc): Conv1D()\n",
              "            (c_proj): Conv1D()\n",
              "            (act): NewGELUActivation()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "    )\n",
              "    (lm_head): Linear(in_features=768, out_features=50257, bias=False)\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (ln): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "  (l3): Linear(in_features=768, out_features=2, bias=True)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
        "print(f\"Trainable parameters: {trainable_params/1000000:.0f}M\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ozaDnGEbWUw8",
        "outputId": "fac9229f-e1fe-4ea4-ab54-bd071ef04901"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Trainable parameters: 124M\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Instantiating training metrics"
      ],
      "metadata": {
        "id": "4jVWeb9SsRsZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Returns the class from the logits returned by the model\n",
        "def get_preds(logits):\n",
        "    return torch.argmax(logits, dim=1)\n",
        "\n",
        "# Returns the number of correct classifications in a batch\n",
        "def get_count_correct(choices, labels):\n",
        "    matches = torch.eq(choices, labels)\n",
        "    return torch.sum(matches).item()"
      ],
      "metadata": {
        "id": "pHv5P20AMFfl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.environ['CUDA_LAUNCH_BLOCKING'] = \"1\""
      ],
      "metadata": {
        "id": "pd86sBiuuLci"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install torchmetrics\n",
        "from torchmetrics import Accuracy, Precision, Recall, F1Score"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yywz1bdFkwq7",
        "outputId": "67a907ba-d2df-423c-a833-2b2d8f68286f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting torchmetrics\n",
            "  Downloading torchmetrics-1.4.0.post0-py3-none-any.whl (868 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m868.8/868.8 kB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>1.20.0 in /usr/local/lib/python3.10/dist-packages (from torchmetrics) (1.25.2)\n",
            "Requirement already satisfied: packaging>17.1 in /usr/local/lib/python3.10/dist-packages (from torchmetrics) (24.0)\n",
            "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from torchmetrics) (2.3.0+cu121)\n",
            "Collecting lightning-utilities>=0.8.0 (from torchmetrics)\n",
            "  Downloading lightning_utilities-0.11.2-py3-none-any.whl (26 kB)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from lightning-utilities>=0.8.0->torchmetrics) (67.7.2)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from lightning-utilities>=0.8.0->torchmetrics) (4.12.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->torchmetrics) (3.14.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->torchmetrics) (1.12.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->torchmetrics) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->torchmetrics) (3.1.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->torchmetrics) (2023.6.0)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch>=1.10.0->torchmetrics)\n",
            "  Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch>=1.10.0->torchmetrics)\n",
            "  Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch>=1.10.0->torchmetrics)\n",
            "  Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch>=1.10.0->torchmetrics)\n",
            "  Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch>=1.10.0->torchmetrics)\n",
            "  Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch>=1.10.0->torchmetrics)\n",
            "  Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "Collecting nvidia-curand-cu12==10.3.2.106 (from torch>=1.10.0->torchmetrics)\n",
            "  Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch>=1.10.0->torchmetrics)\n",
            "  Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch>=1.10.0->torchmetrics)\n",
            "  Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "Collecting nvidia-nccl-cu12==2.20.5 (from torch>=1.10.0->torchmetrics)\n",
            "  Using cached nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl (176.2 MB)\n",
            "Collecting nvidia-nvtx-cu12==12.1.105 (from torch>=1.10.0->torchmetrics)\n",
            "  Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "Requirement already satisfied: triton==2.3.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->torchmetrics) (2.3.0)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.10.0->torchmetrics)\n",
            "  Downloading nvidia_nvjitlink_cu12-12.5.40-py3-none-manylinux2014_x86_64.whl (21.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.3/21.3 MB\u001b[0m \u001b[31m61.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->torchmetrics) (2.1.5)\n",
            "Requirement already satisfied: mpmath<1.4.0,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->torchmetrics) (1.3.0)\n",
            "Installing collected packages: nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, lightning-utilities, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, torchmetrics\n",
            "Successfully installed lightning-utilities-0.11.2 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.20.5 nvidia-nvjitlink-cu12-12.5.40 nvidia-nvtx-cu12-12.1.105 torchmetrics-1.4.0.post0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Training"
      ],
      "metadata": {
        "id": "1tbvSc8tsr_o"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Define hyperparameters\n",
        "LEARNING_RATE = 1e-5\n",
        "EPOCHS = 4\n",
        "\n",
        "# Initialize the loss function\n",
        "loss_fn = torch.nn.CrossEntropyLoss()\n",
        "\n",
        "# Create the optimizer\n",
        "optimizer = torch.optim.Adam(params =  model.parameters(), lr=LEARNING_RATE)\n",
        "\n",
        "# Initialize best loss to a high value\n",
        "best_validation_loss = float('inf')\n",
        "\n",
        "# Metrics for binary classification\n",
        "accuracy = Accuracy(num_classes=2, average='macro', task=\"binary\").to(device)\n",
        "precision = Precision(num_classes=2, average='macro', task=\"binary\").to(device)\n",
        "recall = Recall(num_classes=2, average='macro', task=\"binary\").to(device)\n",
        "f1 = F1Score(num_classes=2, average='macro', task=\"binary\").to(device)\n",
        "\n",
        "# Training loop\n",
        "print(\"training model...\")\n",
        "for epoch in range(EPOCHS):\n",
        "    model.train()\n",
        "    total_train_loss = 0\n",
        "    for batch in train_dataloader:\n",
        "        input_ids = batch['input_ids'].to(device)\n",
        "        attention_mask = batch['attention_mask'].to(device)\n",
        "        lengths = batch['lengths'].to(device)\n",
        "        labels = batch['labels'].to(device)  # tensor: [batch_size]\n",
        "\n",
        "        # Forward pass\n",
        "        logits = model(input_ids, attention_mask, lengths)  # tensor: [batch_size, 2]\n",
        "\n",
        "        # Compute loss\n",
        "        loss = loss_fn(logits, labels)\n",
        "        total_train_loss += loss.item()\n",
        "\n",
        "        # Update metrics\n",
        "        preds = get_preds(logits)\n",
        "        accuracy.update(preds, labels)\n",
        "        precision.update(preds, labels)\n",
        "        recall.update(preds, labels)\n",
        "        f1.update(preds, labels)\n",
        "\n",
        "        # Backward pass and optimization\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "    train_accuracy = accuracy.compute()\n",
        "    train_precision = precision.compute()\n",
        "    train_recall = recall.compute()\n",
        "    train_f1 = f1.compute()\n",
        "    print(f\"Epoch {epoch+1}/{EPOCHS}, Training Loss: {total_train_loss / len(train_dataloader):.4f}, \"\n",
        "          f\"Accuracy: {train_accuracy:.4f}, Precision: {train_precision:.4f}, \"\n",
        "          f\"Recall: {train_recall:.4f}, F1: {train_f1:.4f}\")\n",
        "\n",
        "    # Validation phase\n",
        "    model.eval()\n",
        "    total_validation_loss = 0\n",
        "    accuracy.reset()\n",
        "    precision.reset()\n",
        "    recall.reset()\n",
        "    f1.reset()\n",
        "    with torch.no_grad():\n",
        "        for batch in valid_dataloader:\n",
        "            input_ids = batch['input_ids'].to(device)\n",
        "            attention_mask = batch['attention_mask'].to(device)\n",
        "            lengths = batch['lengths'].to(device)\n",
        "            labels = batch['labels'].to(device)  # tensor: [batch_size]\n",
        "\n",
        "            logits = model(input_ids, attention_mask, lengths) # tensor: [batch_size, 2]\n",
        "\n",
        "            # Calculate loss\n",
        "            loss = loss_fn(logits, labels)\n",
        "            total_validation_loss += loss.item()\n",
        "\n",
        "            # Update metrics\n",
        "            preds = get_preds(logits)\n",
        "            accuracy.update(preds, labels)\n",
        "            precision.update(preds, labels)\n",
        "            recall.update(preds, labels)\n",
        "            f1.update(preds, labels)\n",
        "\n",
        "    avg_validation_loss = total_validation_loss / len(valid_dataloader)\n",
        "    valid_accuracy = accuracy.compute()\n",
        "    valid_precision = precision.compute()\n",
        "    valid_recall = recall.compute()\n",
        "    valid_f1 = f1.compute()\n",
        "    print(f\"Epoch {epoch+1}/{EPOCHS}, Validation Loss: {avg_validation_loss:.4f}, \"\n",
        "          f\"Accuracy: {valid_accuracy:.4f}, Precision: {valid_precision:.4f}, \"\n",
        "          f\"Recall: {valid_recall:.4f}, F1: {valid_f1:.4f}\")\n",
        "\n",
        "    # Save the model if it has the best validation loss so far\n",
        "    if avg_validation_loss < best_validation_loss:\n",
        "        best_validation_loss = avg_validation_loss\n",
        "        torch.save(model.state_dict(), '/content/drive/MyDrive/CS497/Final Project/gpt2_discriminator_Books_state_dict.pth')\n",
        "        print(\"Saved Best Model\")"
      ],
      "metadata": {
        "id": "b-z4BQ0lOOFv",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "02409089-2504-433d-c925-245c98eaa242"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "training model...\n",
            "Epoch 1/4, Training Loss: 0.4629, Accuracy: 0.7795, Precision: 0.7907, Recall: 0.7592, F1: 0.7746\n",
            "Epoch 1/4, Validation Loss: 0.2534, Accuracy: 0.9220, Precision: 0.9035, Recall: 0.9450, F1: 0.9238\n",
            "Saved Best Model\n",
            "Epoch 2/4, Training Loss: 0.2005, Accuracy: 0.9339, Precision: 0.9443, Recall: 0.9221, F1: 0.9330\n",
            "Epoch 2/4, Validation Loss: 0.1566, Accuracy: 0.9404, Precision: 0.9138, Recall: 0.9725, F1: 0.9422\n",
            "Saved Best Model\n",
            "Epoch 3/4, Training Loss: 0.1106, Accuracy: 0.9626, Precision: 0.9614, Recall: 0.9637, F1: 0.9625\n",
            "Epoch 3/4, Validation Loss: 0.1245, Accuracy: 0.9358, Precision: 0.9060, Recall: 0.9725, F1: 0.9381\n",
            "Saved Best Model\n",
            "Epoch 4/4, Training Loss: 0.0720, Accuracy: 0.9726, Precision: 0.9707, Recall: 0.9745, F1: 0.9726\n",
            "Epoch 4/4, Validation Loss: 0.0734, Accuracy: 0.9771, Precision: 0.9815, Recall: 0.9725, F1: 0.9770\n",
            "Saved Best Model\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Testing on the Book reviews dataset"
      ],
      "metadata": {
        "id": "j7kW5Or8sv7_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Assuming model has been loaded and set to the appropriate device\n",
        "model.load_state_dict(torch.load('/content/drive/MyDrive/CS497/Final Project/gpt2_discriminator_Books_state_dict.pth'))\n",
        "model.to(device)\n",
        "model.eval()\n",
        "\n",
        "# Metrics for binary classification\n",
        "accuracy = Accuracy(num_classes=2, average='macro', task=\"binary\").to(device)\n",
        "precision = Precision(num_classes=2, average='macro', task=\"binary\").to(device)\n",
        "recall = Recall(num_classes=2, average='macro', task=\"binary\").to(device)\n",
        "f1 = F1Score(num_classes=2, average='macro', task=\"binary\").to(device)\n",
        "\n",
        "# Testing loop\n",
        "print(\"testing model...\")\n",
        "with torch.no_grad():\n",
        "    for batch in test_dataloader:\n",
        "        input_ids = batch['input_ids'].to(device)\n",
        "        attention_mask = batch['attention_mask'].to(device)\n",
        "        lengths = batch['lengths'].to(device)\n",
        "        labels = batch['labels'].to(device)  # tensor: [batch_size]\n",
        "\n",
        "        logits = model(input_ids, attention_mask, lengths)\n",
        "\n",
        "        # Update metrics\n",
        "        preds = get_preds(logits)\n",
        "        accuracy.update(preds, labels)\n",
        "        precision.update(preds, labels)\n",
        "        recall.update(preds, labels)\n",
        "        f1.update(preds, labels)\n",
        "\n",
        "# Print the fine-tuned test accuracy\n",
        "test_accuracy = accuracy.compute()\n",
        "test_precision = precision.compute()\n",
        "test_recall = recall.compute()\n",
        "test_f1 = f1.compute()\n",
        "print()\n",
        "print(\"Category: Books\")\n",
        "print(f\"Test Accuracy: {test_accuracy:.4f}, Precision: {test_precision:.4f}, \"\n",
        "      f\"Recall: {test_recall:.4f}, F1: {test_f1:.4f}\")\n",
        "print(68*\"-\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ICRqa3DXq93n",
        "outputId": "0e90b814-5f07-42d3-9c78-e87bfa35afb9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "testing model...\n",
            "\n",
            "Category: Books\n",
            "Test Accuracy: 0.9726, Precision: 0.9815, Recall: 0.9636, F1: 0.9725\n",
            "--------------------------------------------------------------------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Evaluating discriminator accuracy on reviews of the other Amazon product review categories to examine generalization"
      ],
      "metadata": {
        "id": "qsZvZpTLiYUp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "categories = ['Clothing Shoes and Jewelry',\n",
        "              'Electronics',\n",
        "              'Home and Kitchen',\n",
        "              'Kindle Store',\n",
        "              'Movies and TV',\n",
        "              'Pet Supplies',\n",
        "              'Sports and Outdoors',\n",
        "              'Tools and Home Improvement',\n",
        "              'Toys and Games']\n",
        "\n",
        "dataloaders = []\n",
        "\n",
        "for category in categories:\n",
        "    category_path = category.replace(' ', '_')\n",
        "    file_name = dataset_path + f\"Generated_Sentences_{category_path}_5.csv\"\n",
        "    df = pd.read_csv(file_name)\n",
        "\n",
        "    _, other_test_df = train_test_split(df, test_size=0.05, stratify=df['label'], random_state=42)\n",
        "\n",
        "    print()\n",
        "    print(f\"Category: {category}\")\n",
        "    print(\"Testing set...\")\n",
        "    print(other_test_df['label'].value_counts(normalize=True))\n",
        "    print(other_test_df.head())\n",
        "    print(55*\"-\")\n",
        "\n",
        "    other_test_dataset = ReviewDataset(other_test_df)\n",
        "    other_test_dataloader = DataLoader(other_test_dataset, batch_size=BATCHSIZE, collate_fn=collate_fn, shuffle=True)\n",
        "    dataloaders.append(other_test_dataloader)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O0wIdQzOyxGu",
        "outputId": "5c302d1b-9318-4f04-a6e1-908643b5595c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Category: Clothing Shoes and Jewelry\n",
            "Testing set...\n",
            "label\n",
            "CG    0.502591\n",
            "OR    0.497409\n",
            "Name: proportion, dtype: float64\n",
            "                          category  rating label  \\\n",
            "3373  Clothing_Shoes_and_Jewelry_5     1.0    CG   \n",
            "2359  Clothing_Shoes_and_Jewelry_5     4.0    CG   \n",
            "652   Clothing_Shoes_and_Jewelry_5     5.0    CG   \n",
            "1017  Clothing_Shoes_and_Jewelry_5     5.0    OR   \n",
            "296   Clothing_Shoes_and_Jewelry_5     4.0    OR   \n",
            "\n",
            "                                                  text_  \n",
            "3373  I'm 5'3\" and 173lbs, 38D.  I wear a medium in ...  \n",
            "2359  The boots are a little tight on the toes, but ...  \n",
            "652   The large was a perfect fit. The wide knee par...  \n",
            "1017  My daughter abusulutley loved them. she says  ...  \n",
            "296   The shoes are great.  They run a little (half ...  \n",
            "-------------------------------------------------------\n",
            "\n",
            "Category: Electronics\n",
            "Testing set...\n",
            "label\n",
            "CG    0.5\n",
            "OR    0.5\n",
            "Name: proportion, dtype: float64\n",
            "           category  rating label  \\\n",
            "3448  Electronics_5     5.0    CG   \n",
            "330   Electronics_5     1.0    OR   \n",
            "2908  Electronics_5     4.0    OR   \n",
            "3790  Electronics_5     4.0    CG   \n",
            "2136  Electronics_5     2.0    OR   \n",
            "\n",
            "                                                  text_  \n",
            "3448  The 25-pack I bought before this one arrived w...  \n",
            "330   So cheaply made it isn't even worth it to out ...  \n",
            "2908  I love the look of the case. That is what drew...  \n",
            "3790  This is an upgrade for my old computer.  I've ...  \n",
            "2136  This case is well designed for the original ve...  \n",
            "-------------------------------------------------------\n",
            "\n",
            "Category: Home and Kitchen\n",
            "Testing set...\n",
            "label\n",
            "CG    0.502463\n",
            "OR    0.497537\n",
            "Name: proportion, dtype: float64\n",
            "                category  rating label  \\\n",
            "1029  Home_and_Kitchen_5     1.0    OR   \n",
            "858   Home_and_Kitchen_5     5.0    OR   \n",
            "1699  Home_and_Kitchen_5     5.0    OR   \n",
            "319   Home_and_Kitchen_5     5.0    CG   \n",
            "2246  Home_and_Kitchen_5     2.0    CG   \n",
            "\n",
            "                                                  text_  \n",
            "1029  rusted after 1 use in dishwasher, was very hap...  \n",
            "858   Love these sheets!  This makes my 2nd purchase...  \n",
            "1699  Been using this over a year now, and have not ...  \n",
            "319   I absolutely love it.  We have had it for a fe...  \n",
            "2246  Cheap. Ripped and torn. If you have the time, ...  \n",
            "-------------------------------------------------------\n",
            "\n",
            "Category: Kindle Store\n",
            "Testing set...\n",
            "label\n",
            "CG    0.50211\n",
            "OR    0.49789\n",
            "Name: proportion, dtype: float64\n",
            "            category  rating label  \\\n",
            "3609  Kindle_Store_5     3.0    CG   \n",
            "3282  Kindle_Store_5     5.0    OR   \n",
            "3463  Kindle_Store_5     5.0    OR   \n",
            "3537  Kindle_Store_5     5.0    OR   \n",
            "4480  Kindle_Store_5     3.0    OR   \n",
            "\n",
            "                                                  text_  \n",
            "3609  Caleb & Amanda's story was a sweet, sweet, fun...  \n",
            "3282  U must read the first book to really understan...  \n",
            "3463  This is a dark, dark book, so be forewarned be...  \n",
            "3537  Why did Mattie want to be Englisch???  This wo...  \n",
            "4480  The Three Laws are:\\n\\nA robot may not injure ...  \n",
            "-------------------------------------------------------\n",
            "\n",
            "Category: Movies and TV\n",
            "Testing set...\n",
            "label\n",
            "CG    0.5\n",
            "OR    0.5\n",
            "Name: proportion, dtype: float64\n",
            "             category  rating label  \\\n",
            "681   Movies_and_TV_5     5.0    CG   \n",
            "1512  Movies_and_TV_5     5.0    OR   \n",
            "1386  Movies_and_TV_5     5.0    OR   \n",
            "2253  Movies_and_TV_5     5.0    OR   \n",
            "447   Movies_and_TV_5     5.0    OR   \n",
            "\n",
            "                                                  text_  \n",
            "681   Right time Right price. It's an entertaining m...  \n",
            "1512  THOR is a great action movie, full of conflict...  \n",
            "1386  This show tells you the history that school do...  \n",
            "2253  Season six with Julia Duffy and Jan Hooks had ...  \n",
            "447   Great collection of movies. I love that they a...  \n",
            "-------------------------------------------------------\n",
            "\n",
            "Category: Pet Supplies\n",
            "Testing set...\n",
            "label\n",
            "CG    0.502347\n",
            "OR    0.497653\n",
            "Name: proportion, dtype: float64\n",
            "            category  rating label  \\\n",
            "716   Pet_Supplies_5     1.0    OR   \n",
            "2820  Pet_Supplies_5     5.0    OR   \n",
            "2582  Pet_Supplies_5     4.0    OR   \n",
            "758   Pet_Supplies_5     5.0    CG   \n",
            "1279  Pet_Supplies_5     2.0    CG   \n",
            "\n",
            "                                                  text_  \n",
            "716   beware...very very small!! very small!! easily...  \n",
            "2820  My cat loves this, and Id also like to add whe...  \n",
            "2582  I really like this light.  The only thing I wi...  \n",
            "758   In-de-structible!  My Lab monster (Cats) has a...  \n",
            "1279  Did not really help my dog. The only thing I d...  \n",
            "-------------------------------------------------------\n",
            "\n",
            "Category: Sports and Outdoors\n",
            "Testing set...\n",
            "label\n",
            "CG    0.5\n",
            "OR    0.5\n",
            "Name: proportion, dtype: float64\n",
            "                   category  rating label  \\\n",
            "227   Sports_and_Outdoors_5     5.0    CG   \n",
            "988   Sports_and_Outdoors_5     5.0    CG   \n",
            "338   Sports_and_Outdoors_5     3.0    OR   \n",
            "2784  Sports_and_Outdoors_5     5.0    OR   \n",
            "51    Sports_and_Outdoors_5     5.0    OR   \n",
            "\n",
            "                                                  text_  \n",
            "227   I like them and will buy more.I bought these f...  \n",
            "988   I love this water bottle.  I have an old stove...  \n",
            "338   It's nice, but the detail, compared to NFL min...  \n",
            "2784  I absolutely love my mat! I used to slip const...  \n",
            "51    Surprisingly accurate darts. A lot of value fo...  \n",
            "-------------------------------------------------------\n",
            "\n",
            "Category: Tools and Home Improvement\n",
            "Testing set...\n",
            "label\n",
            "CG    0.502591\n",
            "OR    0.497409\n",
            "Name: proportion, dtype: float64\n",
            "                          category  rating label  \\\n",
            "2791  Tools_and_Home_Improvement_5     1.0    CG   \n",
            "3333  Tools_and_Home_Improvement_5     3.0    OR   \n",
            "1007  Tools_and_Home_Improvement_5     3.0    CG   \n",
            "2227  Tools_and_Home_Improvement_5     5.0    OR   \n",
            "1706  Tools_and_Home_Improvement_5     5.0    CG   \n",
            "\n",
            "                                                  text_  \n",
            "2791  It's a great little set, and has the quality a...  \n",
            "3333  Good value - but all of the locks are opposite...  \n",
            "1007  It is pretty small overall, but the design is ...  \n",
            "2227  I love these things, I hope they never go chea...  \n",
            "1706  I now have 4 of these. The only problem is tha...  \n",
            "-------------------------------------------------------\n",
            "\n",
            "Category: Toys and Games\n",
            "Testing set...\n",
            "label\n",
            "CG    0.5\n",
            "OR    0.5\n",
            "Name: proportion, dtype: float64\n",
            "              category  rating label  \\\n",
            "1398  Toys_and_Games_5     5.0    CG   \n",
            "1484  Toys_and_Games_5     3.0    OR   \n",
            "908   Toys_and_Games_5     5.0    CG   \n",
            "2505  Toys_and_Games_5     4.0    OR   \n",
            "1740  Toys_and_Games_5     5.0    OR   \n",
            "\n",
            "                                                  text_  \n",
            "1398  My kids love this game! We have had it for a f...  \n",
            "1484  These are good enough, however, there is not a...  \n",
            "908   Super cute you get a lot of fun with it. I wil...  \n",
            "2505  My daughter is loving this set. The wood piece...  \n",
            "1740  My kids love all the Monopolies, especially th...  \n",
            "-------------------------------------------------------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Assuming our fine-tuned model has been loaded and set to the appropriate device\n",
        "model.load_state_dict(torch.load('/content/drive/MyDrive/CS497/Final Project/gpt2_discriminator_Books_state_dict.pth'))\n",
        "model.to(device)\n",
        "model.eval()\n",
        "\n",
        "# Evaluate metrics on the other product categories (excluding Books) on which we did not fine-tune\n",
        "for i, dataloader in enumerate(dataloaders):\n",
        "    # Metrics for binary classification\n",
        "    accuracy = Accuracy(num_classes=2, average='macro', task=\"binary\").to(device)\n",
        "    precision = Precision(num_classes=2, average='macro', task=\"binary\").to(device)\n",
        "    recall = Recall(num_classes=2, average='macro', task=\"binary\").to(device)\n",
        "    f1 = F1Score(num_classes=2, average='macro', task=\"binary\").to(device)\n",
        "\n",
        "    # Testing loop\n",
        "    print()\n",
        "    print(f\"Category: {categories[i]}\")\n",
        "    with torch.no_grad():\n",
        "        for batch in dataloader:\n",
        "            input_ids = batch['input_ids'].to(device)\n",
        "            attention_mask = batch['attention_mask'].to(device)\n",
        "            lengths = batch['lengths'].to(device)\n",
        "            labels = batch['labels'].to(device)  # tensor: [batch_size]\n",
        "\n",
        "            logits = model(input_ids, attention_mask, lengths)\n",
        "\n",
        "            # Update metrics\n",
        "            preds = get_preds(logits)\n",
        "            accuracy.update(preds, labels)\n",
        "            precision.update(preds, labels)\n",
        "            recall.update(preds, labels)\n",
        "            f1.update(preds, labels)\n",
        "\n",
        "    # Print the fine-tuned test accuracy\n",
        "    other_test_accuracy = accuracy.compute()\n",
        "    other_test_precision = precision.compute()\n",
        "    other_test_recall = recall.compute()\n",
        "    other_test_f1 = f1.compute()\n",
        "    print(f\"Test Accuracy: {other_test_accuracy:.4f}, Precision: {other_test_precision:.4f}, \"\n",
        "          f\"Recall: {other_test_recall:.4f}, F1: {other_test_f1:.4f}\")\n",
        "    print(68*\"-\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p9PmtIkcchBr",
        "outputId": "5a9d133e-720f-4738-f210-08c64e120402"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Category: Clothing Shoes and Jewelry\n",
            "Test Accuracy: 0.8860, Precision: 0.9747, Recall: 0.7938, F1: 0.8750\n",
            "--------------------------------------------------------------------\n",
            "\n",
            "Category: Electronics\n",
            "Test Accuracy: 0.9200, Precision: 1.0000, Recall: 0.8400, F1: 0.9130\n",
            "--------------------------------------------------------------------\n",
            "\n",
            "Category: Home and Kitchen\n",
            "Test Accuracy: 0.9113, Precision: 0.9773, Recall: 0.8431, F1: 0.9053\n",
            "--------------------------------------------------------------------\n",
            "\n",
            "Category: Kindle Store\n",
            "Test Accuracy: 0.9662, Precision: 0.9912, Recall: 0.9412, F1: 0.9655\n",
            "--------------------------------------------------------------------\n",
            "\n",
            "Category: Movies and TV\n",
            "Test Accuracy: 0.9556, Precision: 0.9457, Recall: 0.9667, F1: 0.9560\n",
            "--------------------------------------------------------------------\n",
            "\n",
            "Category: Pet Supplies\n",
            "Test Accuracy: 0.9202, Precision: 1.0000, Recall: 0.8411, F1: 0.9137\n",
            "--------------------------------------------------------------------\n",
            "\n",
            "Category: Sports and Outdoors\n",
            "Test Accuracy: 0.8788, Precision: 0.9518, Recall: 0.7980, F1: 0.8681\n",
            "--------------------------------------------------------------------\n",
            "\n",
            "Category: Tools and Home Improvement\n",
            "Test Accuracy: 0.9482, Precision: 1.0000, Recall: 0.8969, F1: 0.9457\n",
            "--------------------------------------------------------------------\n",
            "\n",
            "Category: Toys and Games\n",
            "Test Accuracy: 0.9000, Precision: 0.9524, Recall: 0.8421, F1: 0.8939\n",
            "--------------------------------------------------------------------\n"
          ]
        }
      ]
    }
  ]
}