summariziation_example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3a0e44ff-c46c-4998-88d7-fb3aee89841d",
   "metadata": {},
   "source": [
    "# Text generation models evaluation\n",
    "\n",
    "#### This notebook evaluates several LLMs from Bedrock, HuggingFace, Jumpstart, Bedrock finetuned models\n",
    "#### Instance type used for the evaluation - ml.g4dn.2xlarge or m5.2xlarge, python 3.10\n",
    "#### The metrics evaluated are N-gram matching-based ([ROUGE](https://en.wikipedia.org/wiki/ROUGE_(metric)), [METEOR](https://en.wikipedia.org/wiki/METEOR)) and sematic-based ([BERTScore](https://arxiv.org/abs/1904.09675)) from [FMEval](https://github.com/aws/fmeval/) library (can be further customized), and [BARTScore](https://arxiv.org/abs/2106.11520) using encoder-decoder architecture\n",
    "#### The datasets used is [TweetSumm](https://github.com/guyfe/Tweetsumm) (A Dialog Summarization Dataset for Customer Service, published in EMNLP 21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "68c047c8-0168-45eb-9d59-dda0117ff703",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "# Optional S3 path to upload results to (e.g. s3://yourbucket/results/ ) - Handy to as a way to download results and open html report on a local machine\n",
    "S3_OUTPUT_PATH = None \n",
    "\n",
    "MODELS_TO_EVAL = [] # if empty list will evaluate all the models available. For specific models, mention their ids from the list below, for example [\"anthropic.claude-v2:1\", \"amazon.titan-text-lite-v1\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "eb788c86-761a-4749-927f-737c194e4613",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install --upgrade pip --quiet\n",
    "!pip install -r requirements.txt --quiet"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21f5d06f",
   "metadata": {},
   "source": [
    "### OPEN AI API key\n",
    "This is relevant if you'll be using models from OpenAI\n",
    "\n",
    "- Create a new file called `utils/key.py` in your project directory to store your API key.\n",
    "- Go to your OpenAI account and navigate to \"[View API keys](https://platform.openai.com/account/api-keys).\"\n",
    "- Select \"Create new secret key.\"\n",
    "- Copy the key and insert it into your file `utils/key.py` like this:\n",
    "```\n",
    "OPENAI_API_KEY = 'sk-actualLongKeyGoesHere123'\n",
    "```\n",
    "- Save the changes\n",
    "- IMPORTANT: Do **not** commit `key.py` to source control as will contain your private key. (It should already be in `.gitgnore`.** Review [this information about API safety](https://help.openai.com/en/articles/5112595-best-practices-for-api-key-safety).\n",
    "- Comment out `from utils.key import OPENAI_API_KEY` below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "75cfe9a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "OPENAI_API_KEY = None # uncommenting the line below will override this\n",
    "#from utils.key import OPENAI_API_KEY"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4fbff43-d21a-40c6-b430-3def8ae7c268",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Define bucket config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "11956018-0992-4ec5-bb51-d73b1c017988",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json\n",
    "from pathlib import Path\n",
    "import boto3\n",
    "import glob\n",
    "import shutil\n",
    "import os\n",
    "from os import listdir\n",
    "\n",
    "from fmeval.model_runners.bedrock_model_runner import BedrockModelRunner\n",
    "from fmeval.model_runners.sm_jumpstart_model_runner import JumpStartModelRunner\n",
    "\n",
    "from utils.model_runners.gpt_model_runner import GPTModelConfig, GPTModelRunner\n",
    "from utils.tweetsumm_data_creator import create_train_test_files\n",
    "from utils.model_ranker import create_model_ranking\n",
    "from utils.dashboard_creators.output_viewer_creator import create_response_output_view\n",
    "from utils.dashboard_creators.comparative_dashboard_creator import create_comparive_dashboard\n",
    "from utils.dashboard_creators.data_stats_viewer_creator import create_data_stats_view\n",
    "from utils.dashboard_creators.data_preview_viewer import create_data_preview_view\n",
    "from utils.dashboard_creators.main_html_creator import create_main_html\n",
    "from utils.metrics.bart_score import calculate_bartscore\n",
    "\n",
    "RESULT_FOLDER = \"/tmp/final_result\"\n",
    "if os.path.exists(RESULT_FOLDER):\n",
    "    shutil.rmtree(RESULT_FOLDER)\n",
    "os.mkdir(RESULT_FOLDER)\n",
    "\n",
    "TMP_JSON_FILES = \"/tmp/jsonl_model_files\"\n",
    "if os.path.exists(TMP_JSON_FILES):\n",
    "    shutil.rmtree(TMP_JSON_FILES)\n",
    "os.mkdir(TMP_JSON_FILES)\n",
    "\n",
    "TMP_DATASET_FILES = \"/tmp/dataset_files\"\n",
    "if os.path.exists(TMP_DATASET_FILES):\n",
    "    shutil.rmtree(TMP_DATASET_FILES)\n",
    "os.mkdir(TMP_DATASET_FILES)\n",
    "\n",
    "RESULT_HTML_FOLDER = RESULT_FOLDER + \"/html_files\"\n",
    "if os.path.exists(RESULT_HTML_FOLDER):\n",
    "    shutil.rmtree(RESULT_HTML_FOLDER)\n",
    "os.mkdir(RESULT_HTML_FOLDER)\n",
    "\n",
    "RESULT_IMG_FOLDER = RESULT_FOLDER + \"/imgs\"\n",
    "if os.path.exists(RESULT_IMG_FOLDER):\n",
    "    shutil.rmtree(RESULT_IMG_FOLDER)\n",
    "os.mkdir(RESULT_IMG_FOLDER)\n",
    "\n",
    "from utils.tweetsumm_data_creator import create_train_test_files\n",
    "TEST_FILE_PATH = create_train_test_files(TMP_DATASET_FILES) # creating train and test files\n",
    "print(TEST_FILE_PATH)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "805a915b-4918-4bf4-994a-7cff3701ec91",
   "metadata": {},
   "source": [
    "## List the models to benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c05192f6-a93c-4c76-bb2b-d88e7205426f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Bedrock models\n",
    "models_to_test = {}\n",
    "\n",
    "# Add Bedrock Random text generating model to serve as baseline callibration for the various metrics\n",
    "models_to_test.update({\n",
    "    \"random\" : { \n",
    "        \"model_id\" : \"amazon.titan-text-lite-v1\", \n",
    "        \"platform\" : \"bedrock\",\n",
    "        \"output\" : \"results[0].outputText\", \n",
    "        \"content_template\" : \"{\\\"inputText\\\": $prompt, \\\"textGenerationConfig\\\":  {\\\"maxTokenCount\\\": 100, \\\"stopSequences\\\": [], \\\"temperature\\\": 1.0, \\\"topP\\\": 1.0}}\",\n",
    "        \"prompt_template\" : \"Please ignore the following blob of text and create an unrelated text of around 2 sentences\\n $model_input\\n\"\n",
    "    }\n",
    "})\n",
    "\n",
    "# Add Bedrock Anthropic models in zero-shot\n",
    "models_to_test.update({\n",
    "    \"anthropic.claude-3-sonnet\" : { \n",
    "        \"model_id\" : \"anthropic.claude-3-sonnet-20240229-v1:0\", \n",
    "        \"platform\" : \"bedrock\",\n",
    "        \"output\" : \"content[0].text\", \n",
    "        \"content_template\" : \"{\\\"messages\\\": [{\\\"role\\\": \\\"user\\\", \\\"content\\\": $prompt}], \\\"max_tokens\\\": 100, \\\"anthropic_version\\\": \\\"bedrock-2023-05-31\\\"}\",\n",
    "        \"prompt_template\" : \"Below is a dialog between a customer and an agent. Please provide a short and concise summary of the conversation. The summary should be short and include a single sentence describing the customer's complaint or request, and single sentence of the agent's response or action. Please write the summary in a human readable format. Start you answer directly with the summary without any additional prefix.\\n Specify important and relevant amounts, dates and locations inside the summary. Here is the dialog: <dialog>$model_input</dialog>\"\n",
    "    },\n",
    "    \"anthropic.claude-3-haiku\" : { \n",
    "        \"model_id\" : \"anthropic.claude-3-haiku-20240307-v1:0\", \n",
    "        \"platform\" : \"bedrock\",\n",
    "        \"output\" : \"content[0].text\", \n",
    "        \"content_template\" : \"{\\\"messages\\\": [{\\\"role\\\": \\\"user\\\", \\\"content\\\": $prompt}], \\\"max_tokens\\\": 100, \\\"anthropic_version\\\": \\\"bedrock-2023-05-31\\\"}\",\n",
    "        \"prompt_template\" : \"Below is a dialog between a customer and an agent. Please provide a short and concise summary of the conversation. The summary should be short and include a single sentence describing the customer's complaint or request, and single sentence of the agent's response or action. Please write the summary in a human readable format. Start you answer directly with the summary without any additional prefix.\\n Specify important and relevant amounts, dates and locations inside the summary. Here is the dialog: <dialog>$model_input</dialog>\"\n",
    "    }\n",
    "})\n",
    "\n",
    "# Add Bedrock Amazon Titan models in zero-shot\n",
    "models_to_test.update({\n",
    "    \"amazon.titan-text-lite-v1\" : { \n",
    "        \"model_id\" : \"amazon.titan-text-lite-v1\", \n",
    "        \"platform\" : \"bedrock\",\n",
    "        \"output\" : \"results[0].outputText\", \n",
    "        \"content_template\" : \"{\\\"inputText\\\": $prompt, \\\"textGenerationConfig\\\":  {\\\"maxTokenCount\\\": 100, \\\"stopSequences\\\": [], \\\"temperature\\\": 1.0, \\\"topP\\\": 1.0}}\",\n",
    "        \"prompt_template\" : \"Please provide a short and concise summary of the conversation below. The summary should be short and include a single sentence describing the customer's complaint or request, and single sentence of the agent's response or action. Do not include any additional information that does not appear in the dialog.  Specify important and relevant amounts, dates and locations inside the sentences of the summary. Here is the dialog:\\n$model_input\\n\\nsummary:\\n\"\n",
    "    },\n",
    "    \"amazon.titan-text-express-v1\" :{ \n",
    "        \"model_id\" : \"amazon.titan-text-express-v1\", \n",
    "        \"platform\" : \"bedrock\",\n",
    "        \"output\" : \"results[0].outputText\", \n",
    "        \"content_template\" : \"{\\\"inputText\\\": $prompt, \\\"textGenerationConfig\\\": {\\\"maxTokenCount\\\": 100, \\\"stopSequences\\\": [], \\\"temperature\\\": 1.0, \\\"topP\\\": 1.0}}\",\n",
    "        \"prompt_template\" : \"Please provide a short and concise summary of the conversation below that includes a summary of both the user and the agent.  Specify important and relevant amounts, dates and locations inside the sentences of the summary. Here is the dialog:\\n $model_input\\n\\nsummary:\\n\"\n",
    "    },\n",
    "})\n",
    "\n",
    "# Add Cohere and Llama2 Bedrock models in zero-shot\n",
    "models_to_test.update({\n",
    "    \"cohere.command-light-text-v14\" :{ \n",
    "        \"model_id\" : \"cohere.command-light-text-v14\", \n",
    "        \"platform\" : \"bedrock\",\n",
    "        \"output\" : \"generations[0].text\", \n",
    "        \"content_template\" : \"{\\\"prompt\\\": $prompt, \\\"max_tokens\\\": 100}\",\n",
    "        \"prompt_template\" : \"Please provide a short and concise summary of the conversation below that includes a summary of both the user and the agent.  Specify important and relevant amounts, dates and locations inside the sentences of the summary. Here is the dialog:\\n $model_input\\n\\nsummary:\\n\"\n",
    "    },\n",
    "    \"meta.llama2-13b-chat-v1\" :{ \n",
    "        \"model_id\" : \"meta.llama2-13b-chat-v1\", \n",
    "        \"platform\" : \"bedrock\",\n",
    "        \"output\" : \"generation\", \n",
    "        \"content_template\" : \"{\\\"prompt\\\": $prompt, \\\"max_gen_len\\\": 100, \\\"top_p\\\": 1, \\\"temperature\\\": 1.0}\",\n",
    "        \"prompt_template\" : \"[INST]Please provide a short and concise summary of the conversation below that includes a summary of both the user and the agent.  Specify important and relevant amounts, dates and locations inside the sentences of the summary. Here is the dialog:[/INST]\\n Transcript\\n $model_input \\n\\n Summary:\\n\"\n",
    "    },\n",
    "})\n",
    "\n",
    "# Add various Bedrock models in one-shot\n",
    "models_to_test.update({\n",
    "    \"amazon.titan-text-lite-v1-one-shot\" : { \n",
    "        \"model_id\" : \"amazon.titan-text-lite-v1\", \n",
    "        \"platform\" : \"bedrock\",\n",
    "        \"output\" : \"results[0].outputText\", \n",
    "        \"content_template\" : \"{\\\"inputText\\\": $prompt, \\\"textGenerationConfig\\\":  {\\\"maxTokenCount\\\": 100, \\\"stopSequences\\\": [], \\\"temperature\\\": 1.0, \\\"topP\\\": 1.0}}\",\n",
    "        \"prompt_template\" : \"[INST]Please provide a short and concise summary of the conversation below that includes a summary of both the user and the agent.  Specify important and relevant amounts, dates and locations inside the sentences of the summary. \\n Example Transcript:\\n user: bought a celcus tv from your Finchley store last year in December and it stopped working yesterday - can you repair it or change Your cctv recording from the date we bought it - agent: Can you confirm did you pay cash or card for the telvision? We accept credit/debit card statements as a proof of purchase. Steven user: Yes. I paid by card, I think there were other things I bought with the tv as well , but I remember the price of the television was 175 Actually, I just checked my bank statements and I bought the tv in January 2017 and not dec 2016 and paid for it by card - 175 agent: We would use the bank statements transaction ID to match our till receipts. If you return the television with your credit/debit card...1/2 ...statement our in store colleagues will advise you further. Steven 2/2 user: Great! Thank you. One last question, Ive recycled the Tvs box - is it rwqur Required** agent: As long as you've got proof of purchase you'll be fine Dimitar! Ewan.\\n Summary: Customer is asking to repair or change the television which is not working. Agent updated to return the television with their credit/debit card.\\n\\n [/INST] </s><s>[INST]\\n Transcript:\\n $model_input [/INST]\\n Summary:\"\n",
    "    },\n",
    "    \"meta.llama2-13b-chat-v1-one-shot\" :{ \n",
    "        \"model_id\" : \"meta.llama2-13b-chat-v1\", \n",
    "        \"platform\" : \"bedrock\",\n",
    "        \"output\" : \"generation\", \n",
    "        \"content_template\" : \"{\\\"prompt\\\": $prompt, \\\"max_gen_len\\\": 100, \\\"top_p\\\": 1, \\\"temperature\\\": 1.0}\",\n",
    "        \"prompt_template\" : \"[INST] <<SYS>> Please provide a short and concise summary of the conversation below that includes a summary of both the user and the agent.  Specify important and relevant amounts, dates and locations inside the sentences of the summary.<</SYS> \\n Example Transcript:\\n user: bought a celcus tv from your Finchley store last year in December and it stopped working yesterday - can you repair it or change Your cctv recording from the date we bought it - agent: Can you confirm did you pay cash or card for the telvision? We accept credit/debit card statements as a proof of purchase. Steven user: Yes. I paid by card, I think there were other things I bought with the tv as well , but I remember the price of the television was 175 Actually, I just checked my bank statements and I bought the tv in January 2017 and not dec 2016 and paid for it by card - 175 agent: We would use the bank statements transaction ID to match our till receipts. If you return the television with your credit/debit card...1/2 ...statement our in store colleagues will advise you further. Steven 2/2 user: Great! Thank you. One last question, Ive recycled the Tvs box - is it rwqur Required** agent: As long as you've got proof of purchase you'll be fine Dimitar! Ewan.\\n Summary: Customer is asking to repair or change the television which is not working. Agent updated to return the television with their credit/debit card.\\n\\n [/INST] </s><s>[INST]\\n Transcript:\\n $model_input [/INST] Summary:\"\n",
    "    },\n",
    "    \"cohere.command-light-text-v14-one-shot\" :{ \n",
    "        \"model_id\" : \"cohere.command-light-text-v14\", \n",
    "        \"platform\" : \"bedrock\",\n",
    "        \"output\" : \"generations[0].text\", \n",
    "        \"content_template\" : \"{\\\"prompt\\\": $prompt, \\\"max_tokens\\\": 100}\",\n",
    "        \"prompt_template\" : \"Please provide a short and concise summary of the conversation below that includes a summary of both the user and the agent.  Specify important and relevant amounts, dates and locations inside the sentences of the summary.\\n\\n Example Transcript:\\n user: bought a celcus tv from your Finchley store last year in December and it stopped working yesterday - can you repair it or change Your cctv recording from the date we bought it - agent: Can you confirm did you pay cash or card for the telvision? We accept credit/debit card statements as a proof of purchase. Steven user: Yes. I paid by card, I think there were other things I bought with the tv as well , but I remember the price of the television was 175 Actually, I just checked my bank statements and I bought the tv in January 2017 and not dec 2016 and paid for it by card - 175 agent: We would use the bank statements transaction ID to match our till receipts. If you return the television with your credit/debit card...1/2 ...statement our in store colleagues will advise you further. Steven 2/2 user: Great! Thank you. One last question, Ive recycled the Tvs box - is it rwqur Required** agent: As long as you've got proof of purchase you'll be fine Dimitar! Ewan.\\n Summary: Customer is asking to repair or change the television which is not working. Agent updated to return the television with their credit/debit card.\\n\\nTranscript:\\n $model_input\\n Summary:\"\n",
    "    },\n",
    "    \"amazon.titan-text-express-v1-one-shot\" :{ \n",
    "        \"model_id\" : \"amazon.titan-text-express-v1\", \n",
    "        \"platform\" : \"bedrock\",\n",
    "        \"output\" : \"results[0].outputText\", \n",
    "        \"content_template\" : \"{\\\"inputText\\\": $prompt, \\\"textGenerationConfig\\\": {\\\"maxTokenCount\\\": 100, \\\"stopSequences\\\": [], \\\"temperature\\\": 1.0, \\\"topP\\\": 1.0}}\",\n",
    "        \"prompt_template\" : \"Please provide a short and concise summary of the conversation below that includes a summary of both the user and the agent.  Specify important and relevant amounts, dates and locations inside the sentences of the summary.\\n\\n Example Transcript:\\n user: bought a celcus tv from your Finchley store last year in December and it stopped working yesterday - can you repair it or change Your cctv recording from the date we bought it - agent: Can you confirm did you pay cash or card for the telvision? We accept credit/debit card statements as a proof of purchase. Steven user: Yes. I paid by card, I think there were other things I bought with the tv as well , but I remember the price of the television was 175 Actually, I just checked my bank statements and I bought the tv in January 2017 and not dec 2016 and paid for it by card - 175 agent: We would use the bank statements transaction ID to match our till receipts. If you return the television with your credit/debit card...1/2 ...statement our in store colleagues will advise you further. Steven 2/2 user: Great! Thank you. One last question, Ive recycled the Tvs box - is it rwqur Required** agent: As long as you've got proof of purchase you'll be fine Dimitar! Ewan.\\n Summary: Customer is asking to repair or change the television which is not working. Agent updated to return the television with their credit/debit card.\\n\\nTranscript:\\n $model_input\\n Summary:\"\n",
    "    },\n",
    "})\n",
    "\n",
    "# Add OpenAI models in zero-shot\n",
    "models_to_test.update({\n",
    "    \"gpt.3.5-turbu-0125\" :{ \n",
    "        \"model_id\" : \"gpt-3.5-turbo-0125\", \n",
    "        \"api_key\" : OPENAI_API_KEY,\n",
    "        \"platform\" : \"openai\",\n",
    "        \"temperature\" : 1,\n",
    "        \"top_p\" : 1,\n",
    "        \"max_tokens\" : 100,\n",
    "        \"prompt_template\" : \"Please provide a short and concise summary of the conversation below that includes a summary of both the user and the agent.  Specify important and relevant amounts, dates and locations inside the sentences of the summary.\\n Transcript:\\n $model_input \\n Summary:\\n\"\n",
    "    }\n",
    "})\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb20053f-e76e-4ffc-b94d-a6589409776c",
   "metadata": {},
   "source": [
    "## Adding your own custom models\n",
    "In case you wish to add custom model, simply create custom model runner. For example, see custom model runner which wraps GPT-3.5 in the folder utils/model_runners/gpt_model_runner.py \n",
    "\n",
    "\n",
    "## Adding finetuned models\n",
    "In case you wish to add Bedrock finetuned model: \n",
    "1. First finetune a model (for details on finetuning on Berdrock visit https://aws.amazon.com/blogs/aws/customize-models-in-amazon-bedrock-with-your-own-data-using-fine-tuning-and-continued-pre-training/).\n",
    "2. Once training completed, from Bedrock copy the ARN from Bedrock 'provisioned throughput' dashboard and paste it as the model_id. A finetuning training set is provided. For more details see documentation\n",
    "3. Add to the model_dict in the cell above the configuration of your finetuned model as follows:\n",
    "\n",
    "<code>\n",
    "{\n",
    "    \"finetuned_amazon.titan-text-lite-v1\" : {\n",
    "    \"platform\":\"bedrock\",\n",
    "    \"model_id\": \"arn:aws:bedrock:us-east-1:333333333:provisioned-model/879asd6s75\",\n",
    "    \"output\": \"results[0].outputText\",\n",
    "    \"content_template\": {\"inputText\": $prompt, \"textGenerationConfig\":  {\"maxTokenCount\": 100, \"stopSequences\": [], \"temperature\": 1.0, \"topP\": 1.0}},\n",
    "    \"prompt_template\": \"YOUR PROMPT HERE\"\n",
    "    }\n",
    "}\n",
    "</code>\n",
    "\n",
    "\n",
    "## Adding Jumpstart models\n",
    "Example for evaluation Mistral-7B-Instruct from Jumpstart:\n",
    "1. Go to Jumpstart (press home button -> Jumpstart)\n",
    "2. Search in the bar for Mistral-7B-Instruct\n",
    "3. Click deploy from the model card (don't forget to close the endpoint once you done from SageMaker->inference endpoints)\n",
    "4. Add the following to the models list\n",
    "<code>\n",
    "{\n",
    "    \"platform\":\"jumpstart\",\n",
    "    \"model_id\": \"huggingface-llm-mistral-7b-instruct\",\n",
    "    \"endpoint_name\": \"jumpstart-dft-hf-llm-mistral-7b-instruct\",\n",
    "    \"model_version\": \"*\",\n",
    "    \"output\": \"[0].generated_text\",\n",
    "    \"content_template\":\"{\\\"inputs\\\": $prompt, \\\"parameters\\\": {\\\"do_sample\\\": false, \\\"max_new_tokens\\\": 100}}\",\n",
    "    \"prompt_template\": \"YOUR PROMPT HERE\"\n",
    "}\n",
    "</code>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac160609-dfb3-4eb5-bac0-753053f27184",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Creating ModelRunner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a9466069-430a-4f71-80a2-c0c6e3b4e918",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from utils.model_runners.bedrock_counting_runner import CountingBedrockModelRunner\n",
    "\n",
    "\n",
    "def get_models_to_eval():\n",
    "    if len(MODELS_TO_EVAL) == 0:\n",
    "        return list(models_to_test.keys())\n",
    "    return MODELS_TO_EVAL\n",
    "\n",
    "models = dict()        \n",
    "for fm in get_models_to_eval():  \n",
    "    \n",
    "    data = models_to_test[fm]\n",
    "    platform = data['platform']\n",
    "    \n",
    "    if platform == \"bedrock\":\n",
    "        runner = CountingBedrockModelRunner(model_id=data[\"model_id\"], output=data[\"output\"], content_template=data[\"content_template\"].replace(\"'\",\"\\\"\"),metrics_folder = TMP_JSON_FILES, model_key = fm)\n",
    "    elif platform == \"jumpstart\":\n",
    "        runner = JumpStartModelRunner(endpoint_name=data[\"endpoint_name\"], model_id=data[\"model_id\"], model_version=data[\"model_version\"], output=data[\"output\"].replace(\"'\",\"\\\"\"), content_template=data[\"content_template\"].replace(\"'\",\"\\\"\"))\n",
    "    elif platform == \"openai\":\n",
    "        if OPENAI_API_KEY:\n",
    "            runner = GPTModelRunner(GPTModelConfig(model_id=data[\"model_id\"], api_key=data[\"api_key\"], temperature=data[\"temperature\"], top_p=data[\"top_p\"], max_tokens=data[\"max_tokens\"]),metrics_folder = TMP_JSON_FILES, model_key = fm)\n",
    "        else:\n",
    "            print(\"Skipping OpenAI models - Cannot run without an API key\")\n",
    "            continue\n",
    "        \n",
    "    models[fm] = { \"model_runner\": runner, \"prompt_template\": data[\"prompt_template\"]}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6198a20-9a87-4a16-af6d-35733f9845c7",
   "metadata": {},
   "source": [
    "## Evaluation run\n",
    "Evaluating METEOR, ROUGE, and BERTscore using FMEval library (https://github.com/aws/fmeval). This library is also used by Bedrock when finetuning or evaluating models.\n",
    "\n",
    "#### Note - if while running this cell you encounter the message - \"Error displaying widget: model not found\" in the evaluation phase...\", simply ignore it. It relates to the UI and does not effect the evaluation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7ec118e5-b099-49af-998a-1cacf6a7664b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from fmeval.data_loaders.data_config import DataConfig\n",
    "from fmeval.constants import MIME_TYPE_JSONLINES\n",
    "from fmeval.eval_algorithms.summarization_accuracy import SummarizationAccuracy, SummarizationAccuracyConfig\n",
    "from utils.model_runners.pricing_calculator import PricingCalculator\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "os.environ[\"PARALLELIZATION_FACTOR\"] = \"1\" # will use a single workder for FMEval\n",
    "TMP_JSON_FILES = \"/tmp/jsonl_model_files\"\n",
    "if os.path.exists(TMP_JSON_FILES):\n",
    "    shutil.rmtree(TMP_JSON_FILES)\n",
    "os.mkdir(TMP_JSON_FILES)\n",
    "\n",
    "models_scores = dict()\n",
    "models_usage = dict()\n",
    "models_to_eval = get_models_to_eval()\n",
    "for model_id in models_to_eval:\n",
    "    print(f\"### Starting model {model_id} evaluation\")\n",
    "    if not model_id in models:\n",
    "        print(f\"###model {model_id} doesn't have a valid/complete entry in the model list\")\n",
    "        continue\n",
    "    model = models[model_id]\n",
    "    config = DataConfig(\n",
    "        dataset_name=f\"data\",\n",
    "        dataset_uri=TEST_FILE_PATH,\n",
    "        dataset_mime_type=MIME_TYPE_JSONLINES,\n",
    "        model_input_location=\"document\",\n",
    "        target_output_location=\"summary\"\n",
    "    )\n",
    "\n",
    "    model_runner = model['model_runner']\n",
    "    eval_algo = SummarizationAccuracy(SummarizationAccuracyConfig())\n",
    "    eval_output = eval_algo.evaluate(model=model_runner, \n",
    "                                     dataset_config=config,\n",
    "                                     prompt_template=model[\"prompt_template\"],\n",
    "                                     num_records=10,\n",
    "                                     save=True)\n",
    "\n",
    "    scores = dict()\n",
    "    for i in eval_output[0].dataset_scores:\n",
    "        scores[i.name] = i.value\n",
    "    \n",
    "    models_scores[model_id] = scores\n",
    "    models_usage[model_id] = PricingCalculator.read_model_score_aggregate(model_id, TMP_JSON_FILES)\n",
    "    shutil.move('/tmp/eval_results/summarization_accuracy_data.jsonl', f'{TMP_JSON_FILES}/{model_id}_metrics.jsonl')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ecf0a7ea-0ae1-423a-949a-6526b5b97ef9",
   "metadata": {},
   "source": [
    "## Calculate BARTscore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5e019170-e364-4c54-a030-b00207bcc5e8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "### Metrics to calc\n",
    "# BARTscore - for more details https://github.com/neulab/BARTScore/blob/main/README.md\n",
    "CALC_BARTSCORE = True\n",
    "\n",
    "PATH_TO_FINETUNED_BART = \"\" # if left empty will use vanilla BART. If you wish to load the finetuned BART, go to BARTscore's github, download the bart_score.pth (appear on the README) and provide the path here\n",
    "if CALC_BARTSCORE:\n",
    "    calculate_bartscore(TMP_JSON_FILES, models_scores, PATH_TO_FINETUNED_BART)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "604b4580-546c-48ca-be94-fffabd1cc280",
   "metadata": {},
   "source": [
    "## Create Leaderboard Report HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0c6485fe-e5b4-41b1-8323-470b963c8c20",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from utils.model_ranker import create_model_ranking\n",
    "create_response_output_view(RESULT_HTML_FOLDER, TMP_JSON_FILES, models_scores)\n",
    "create_comparive_dashboard(RESULT_HTML_FOLDER, TMP_JSON_FILES)\n",
    "create_data_stats_view(TEST_FILE_PATH, RESULT_IMG_FOLDER)\n",
    "create_data_preview_view(TEST_FILE_PATH, RESULT_HTML_FOLDER)\n",
    "main_html_filename = create_main_html(RESULT_FOLDER, models_scores, models_usage)\n",
    "\n",
    "print(f\"Created leaderboard in: {main_html_filename}\")\n",
    "\n",
    "# archive entire report\n",
    "from datetime import datetime\n",
    "today = datetime.now()\n",
    "my_datetime = str(today.strftime(\"%d-%m-%Y_%H-%M-%S\"))\n",
    "zip_filename_fullpath = shutil.make_archive(f\"/tmp/{my_datetime}\", 'zip', \"/tmp/final_result\")\n",
    "zip_filename = zip_filename_fullpath.split(\"/\")[-1] # filename without folders\n",
    "print(f\"Archived report in: {zip_filename_fullpath}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36570a56-4ba5-4143-aac0-b4ba99afce65",
   "metadata": {},
   "source": [
    "## Upload Report to S3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75e36a98-6c1d-49ed-8f43-e2992be1e96d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "if S3_OUTPUT_PATH: # if defined S3\n",
    "    s3_key = f\"{S3_OUTPUT_PATH}/{zip_filename}\"\n",
    "    !aws s3 cp {zip_filename_fullpath} {s3_key}\n",
    "    print(f\"Uploaded to: {s3_key}\")\n",
    "else:\n",
    "    print(f\"No S3_OUTPUT_PATH set, not uploading {zip_filename}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "983658a1",
   "metadata": {},
   "source": [
    "## Viewing results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ae6175ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "if S3_OUTPUT_PATH:\n",
    "    print(f'If running on a *remote* machine to view the results on your local computer copy-paste these commands in your terminal:\\n\\\n",
    "    aws s3 cp {s3_key} /tmp/{zip_filename}\\n\\\n",
    "    cd /tmp\\n\\\n",
    "    unzip -d {zip_filename.replace(\".zip\",\"\")} {zip_filename}\\n\\\n",
    "    open /tmp/{zip_filename.replace(\".zip\",\"\")}/index.html\\n')\n",
    "\n",
    "print(f'If running on a *local* machine copy-paste these commands in your terminal:\\n\\\n",
    "    open {main_html_filename}')"
   ]
  }
 ],
 "metadata": {
  "availableInstances": [
   {
    "_defaultOrder": 0,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.t3.medium",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 1,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.t3.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 2,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.t3.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 3,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.t3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 4,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 5,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 6,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 7,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 8,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 9,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 10,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 11,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 12,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5d.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 13,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5d.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 14,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5d.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 15,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5d.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 16,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5d.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 17,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5d.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 18,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5d.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 19,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 20,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": true,
    "memoryGiB": 0,
    "name": "ml.geospatial.interactive",
    "supportedImageNames": [
     "sagemaker-geospatial-v1-0"
    ],
    "vcpuNum": 0
   },
   {
    "_defaultOrder": 21,
    "_isFastLaunch": true,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.c5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 22,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.c5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 23,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.c5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 24,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.c5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 25,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 72,
    "name": "ml.c5.9xlarge",
    "vcpuNum": 36
   },
   {
    "_defaultOrder": 26,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 96,
    "name": "ml.c5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 27,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 144,
    "name": "ml.c5.18xlarge",
    "vcpuNum": 72
   },
   {
    "_defaultOrder": 28,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.c5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 29,
    "_isFastLaunch": true,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g4dn.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 30,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g4dn.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 31,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g4dn.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 32,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g4dn.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 33,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g4dn.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 34,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g4dn.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 35,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 61,
    "name": "ml.p3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 36,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 244,
    "name": "ml.p3.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 37,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 488,
    "name": "ml.p3.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 38,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.p3dn.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 39,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.r5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 40,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.r5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 41,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.r5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 42,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.r5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 43,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.r5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 44,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.r5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 45,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.r5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 46,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.r5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 47,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 48,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 49,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 50,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 51,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 52,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 53,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.g5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 54,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.g5.48xlarge",
    "vcpuNum": 192
   },
   {
    "_defaultOrder": 55,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 56,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4de.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 57,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.trn1.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 58,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.trn1.32xlarge",
    "vcpuNum": 128
   },
   {
    "_defaultOrder": 59,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.trn1n.32xlarge",
    "vcpuNum": 128
   }
  ],
  "instance_type": "ml.g4dn.xlarge",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}