gemini15-long-context-competition.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1cb77f6a",
   "metadata": {
    "papermill": {
     "duration": 0.005037,
     "end_time": "2024-11-12T21:40:31.695137",
     "exception": false,
     "start_time": "2024-11-12T21:40:31.690100",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Define functions to scrape websites"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8e67422",
   "metadata": {
    "papermill": {
     "duration": 0.00405,
     "end_time": "2024-11-12T21:40:31.703648",
     "exception": false,
     "start_time": "2024-11-12T21:40:31.699598",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Install the required dependencies to scrape websites"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "dda41ecc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-12T21:40:31.713925Z",
     "iopub.status.busy": "2024-11-12T21:40:31.713497Z",
     "iopub.status.idle": "2024-11-12T21:40:56.100389Z",
     "shell.execute_reply": "2024-11-12T21:40:56.099050Z"
    },
    "papermill": {
     "duration": 24.395358,
     "end_time": "2024-11-12T21:40:56.103239",
     "exception": false,
     "start_time": "2024-11-12T21:40:31.707881",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: beautifulsoup4 in /opt/conda/lib/python3.10/site-packages (4.12.3)\r\n",
      "Requirement already satisfied: soupsieve>1.2 in /opt/conda/lib/python3.10/site-packages (from beautifulsoup4) (2.5)\r\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (2.32.3)\r\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests) (3.3.2)\r\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests) (3.7)\r\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests) (1.26.18)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests) (2024.8.30)\r\n"
     ]
    }
   ],
   "source": [
    "!pip install beautifulsoup4\n",
    "!pip install requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "128dcaa0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-12T21:40:56.114917Z",
     "iopub.status.busy": "2024-11-12T21:40:56.114513Z",
     "iopub.status.idle": "2024-11-12T21:40:56.420749Z",
     "shell.execute_reply": "2024-11-12T21:40:56.419633Z"
    },
    "papermill": {
     "duration": 0.315623,
     "end_time": "2024-11-12T21:40:56.423534",
     "exception": false,
     "start_time": "2024-11-12T21:40:56.107911",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import os\n",
    "from urllib.parse import urljoin\n",
    "from collections import deque\n",
    "\n",
    "VALID_GET_RESPONSE = 200\n",
    "\n",
    "\n",
    "def soup_page(url: str) -> BeautifulSoup:\n",
    "    \"\"\"\n",
    "    Function to make a HTTP request by a given url\n",
    "    \"\"\"\n",
    "    response = requests.get(url)\n",
    "    if response.status_code == VALID_GET_RESPONSE:\n",
    "        soup = BeautifulSoup(response.text, \"html.parser\")\n",
    "    else:\n",
    "        soup = BeautifulSoup(\"\")\n",
    "    return soup\n",
    "\n",
    "\n",
    "def get_links(soup: BeautifulSoup, url: str, parentFolder: str) -> list:\n",
    "    \"\"\"\n",
    "    Function to get all links from a page\n",
    "    \"\"\"\n",
    "    links = [\n",
    "        a[\"href\"] for a in soup.find_all(\"a\", href=True) if parentFolder in a[\"href\"] and not \"contact-us\" in a[\"href\"]\n",
    "    ]\n",
    "    return links\n",
    "\n",
    "\n",
    "def download_page(soup: BeautifulSoup, url: str, savePath: str) -> str:\n",
    "    \"\"\"\n",
    "    Function to download a page\n",
    "    \"\"\"\n",
    "    paragraphsList = [p.get_text() for p in soup.find_all(\"p\")]\n",
    "    pageTitle = [t.get_text() for t in soup.find_all(\"title\")]\n",
    "    if paragraphsList:\n",
    "        siteData = {\"title\": pageTitle, \"url\": url, \"content\": paragraphsList}\n",
    "\n",
    "        urlPath = url.replace(\"https://\", \"\").replace(\"http://\", \"\").replace(\"/\", \"_\")\n",
    "        filePath = os.path.join(savePath, urlPath + \".json\")\n",
    "        write_json_from_data(siteData, filePath)\n",
    "        return filePath\n",
    "    else:\n",
    "        return \"\"\n",
    "\n",
    "\n",
    "def write_json_from_data(data: dict, filePath: str, indentSize: int = 4) -> None:\n",
    "    \"\"\"\n",
    "    Function to write a json file with its data\n",
    "    \"\"\"\n",
    "    with open(filePath, \"w\") as f:\n",
    "        json.dump(data, f, ensure_ascii=False, indent=indentSize)\n",
    "    return\n",
    "\n",
    "\n",
    "def download_all_pages(\n",
    "    rootUrl: str, parentFolder: str, savePath: str, maxPages: int = 100\n",
    ") -> list[str]:\n",
    "    \"\"\"\n",
    "    Recursive function to download pages and subpages\n",
    "    \"\"\"\n",
    "    visitedUrls = set()\n",
    "    stack = deque([rootUrl])\n",
    "\n",
    "    jsonFiles = []\n",
    "    while stack and len(jsonFiles) < maxPages:\n",
    "        url = stack.pop()\n",
    "\n",
    "        if url in visitedUrls:\n",
    "            continue\n",
    "        visitedUrls.add(url)\n",
    "\n",
    "        soup = soup_page(url)\n",
    "        jsonFileFromSite = download_page(soup, url, savePath)\n",
    "        print(f\"JSON file {jsonFileFromSite} created from url {url}\")\n",
    "        jsonFiles.append(jsonFileFromSite)\n",
    "\n",
    "        links = get_links(soup, url, parentFolder)\n",
    "        for link in links:\n",
    "            fullLink = urljoin(url, link)\n",
    "            if fullLink not in visitedUrls and \"http\" in fullLink:\n",
    "                stack.append(fullLink)\n",
    "\n",
    "    return jsonFiles\n",
    "\n",
    "\n",
    "def most_common_sentences_in_file(\n",
    "    jsonFilePath: str, alreadyCommonWords: set = set(), frequencyThreshold: int = 1\n",
    ") -> list[str]:\n",
    "    with open(jsonFilePath, \"r\") as f:\n",
    "        data = json.load(f)\n",
    "\n",
    "    if type(data) == list:\n",
    "        contentList = [c.get(\"content\", \"\") for c in data]\n",
    "        content = []\n",
    "        for contentPage in contentList:\n",
    "            content.extend(contentPage)\n",
    "    else:\n",
    "        content = data.get(\"content\", \"\")\n",
    "\n",
    "    frequencies = {}\n",
    "    if content:\n",
    "        for sentence in content:\n",
    "            frequencies[sentence] = frequencies.get(sentence, 0) + 1\n",
    "\n",
    "    for sentence, freq in frequencies.items():\n",
    "        if freq > frequencyThreshold:\n",
    "            alreadyCommonWords.add(sentence)\n",
    "    return alreadyCommonWords\n",
    "\n",
    "\n",
    "def list_json_files_in_folder(\n",
    "    jsonFilesFolder: str, jsonFilesToExclude: str | list[str]\n",
    ") -> list[str]:\n",
    "    if type(jsonFilesToExclude) == str:\n",
    "        jsonFilesToExclude = [jsonFilesToExclude]\n",
    "\n",
    "    filesToExclude = set(jsonFilesToExclude)\n",
    "    fileList = [\n",
    "        os.path.join(jsonFilesFolder, file)\n",
    "        for file in os.listdir(jsonFilesFolder)\n",
    "        if file.endswith(\".json\") and file not in filesToExclude\n",
    "    ]\n",
    "    return fileList\n",
    "\n",
    "\n",
    "def clean_json_file(\n",
    "    jsonFilePath: str, mostCommonRows: set[str] = set(), overwrite: bool = True\n",
    ") -> str:\n",
    "    with open(jsonFilePath, \"r\") as f:\n",
    "        data = json.load(f)\n",
    "\n",
    "    if type(data) == list:\n",
    "        for i, page in enumerate(data):\n",
    "            indexesToRemove = set()\n",
    "            for j, row in enumerate(page.get(\"content\", \"\")):\n",
    "                if row in mostCommonRows:\n",
    "                    indexesToRemove.add(j)\n",
    "\n",
    "            data[i][\"content\"][:] = [\n",
    "                c for k, c in enumerate(data[i][\"content\"]) if not k in indexesToRemove\n",
    "            ]\n",
    "    else:\n",
    "        indexesToRemove = set()\n",
    "        for i, row in enumerate(data.get(\"content\", \"\")):\n",
    "            if row in mostCommonRows:\n",
    "                indexesToRemove.add(i)\n",
    "\n",
    "        data[\"content\"][:] = [\n",
    "            c for k, c in enumerate(data[\"content\"]) if k not in indexesToRemove\n",
    "        ]\n",
    "\n",
    "    if overwrite:\n",
    "        jsonCleanedFilePath = jsonFilePath\n",
    "    else:\n",
    "        jsonCleanedFilePath = jsonFilePath.replace(\".json\", \"_cleaned.json\")\n",
    "\n",
    "    write_json_from_data(data, jsonCleanedFilePath)\n",
    "    return jsonCleanedFilePath\n",
    "\n",
    "\n",
    "def clean_json_files(\n",
    "    jsonFilesFolder: str, filesToExclude: str | list[str], overwrite: bool = True\n",
    ") -> list[str]:\n",
    "    \"\"\"\n",
    "    Function to remove unneded text from a json file containing web page content\n",
    "    \"\"\"\n",
    "    fileList = list_json_files_in_folder(jsonFilesFolder, filesToExclude)\n",
    "\n",
    "    mostCommonRows = set()  # it is possible to define custom common words here\n",
    "    for file in fileList:\n",
    "        mostCommonRows = most_common_sentences_in_file(file, mostCommonRows)\n",
    "\n",
    "    cleanedFiles = []\n",
    "    for file in fileList:\n",
    "        jsonCleanedFilePath = clean_json_file(file, mostCommonRows, overwrite)\n",
    "        cleanedFiles.append(jsonCleanedFilePath)\n",
    "    return cleanedFiles\n",
    "\n",
    "\n",
    "def merge_json_files(jsonFilesFolder: str, targetFile: str = \"_merged.json\") -> str:\n",
    "    \"\"\"\n",
    "    Function to merge multiple json files into a single one\n",
    "    \"\"\"\n",
    "    fileList = list_json_files_in_folder(jsonFilesFolder, targetFile)\n",
    "    if fileList:\n",
    "        resultFileContent = []\n",
    "        for file in fileList:\n",
    "            with open(file, \"r\") as f:\n",
    "                jsonData = json.load(f)\n",
    "                resultFileContent.append(jsonData)\n",
    "\n",
    "        resultFilePath = os.path.join(jsonFilesFolder, targetFile)\n",
    "        write_json_from_data(resultFileContent, resultFilePath)\n",
    "        return resultFilePath\n",
    "    return \"\"\n",
    "\n",
    "\n",
    "def json_to_txt(savePath: str, jsonFilePath: str, targetName: str = \"\") -> str:\n",
    "    if targetName == \"\":\n",
    "        targetName = os.path.basename(jsonFilePath).replace(\".json\", \".txt\")\n",
    "\n",
    "    with open(jsonFilePath, \"r\") as f:\n",
    "        data = json.load(f)\n",
    "\n",
    "    resultFilePath = os.path.join(savePath, targetName)\n",
    "    with open(resultFilePath, \"w\") as f:\n",
    "        for numPage, webpage in enumerate(data):\n",
    "            pageTitle = webpage.get(\"title\", \"\")\n",
    "            f.write(\n",
    "                f'The page number {numPage+1} with name \"{pageTitle[0]}\" has the following content:\\n'\n",
    "            )\n",
    "\n",
    "            pageContent = webpage.get(\"content\", \"\")\n",
    "            for row in pageContent:\n",
    "                f.write(f\"{row} \\n\")\n",
    "\n",
    "            f.write(\n",
    "                f'The page number {numPage+1} with name \"{pageTitle[0]}\" ends here.\\n\\n'\n",
    "            )\n",
    "\n",
    "    return resultFilePath\n",
    "\n",
    "\n",
    "def text_from_file(filePath: str) -> str | list[str]:\n",
    "    with open(filePath, \"r\", encoding=\"utf-8\") as f:\n",
    "        textString = f.read()\n",
    "        f.seek(0)\n",
    "        textList = f.readlines()\n",
    "    return textString, textList\n",
    "\n",
    "\n",
    "def get_text_from_webpages(\n",
    "    root: str, parentFolder: str, savePath: str = \"crawled\", numPages: int = 5\n",
    ") -> str:\n",
    "    os.makedirs(savePath, exist_ok=True)\n",
    "\n",
    "    jsonFiles = download_all_pages(root, parentFolder, savePath, numPages)\n",
    "\n",
    "    mergedJsonName = \"_merged.json\"\n",
    "    jsonFilesClean = clean_json_files(savePath, mergedJsonName)\n",
    "    mergedJsonPath = merge_json_files(savePath, mergedJsonName)\n",
    "\n",
    "    mostCommonWordsMergedJson = most_common_sentences_in_file(\n",
    "        mergedJsonPath, frequencyThreshold=len(jsonFilesClean) - 1\n",
    "    )\n",
    "    mergedJsonPath = clean_json_file(mergedJsonPath, mostCommonWordsMergedJson)\n",
    "\n",
    "    plainTextFile = json_to_txt(savePath, mergedJsonPath)\n",
    "    plainText, textString = text_from_file(plainTextFile)\n",
    "    return plainText"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d0a20814",
   "metadata": {
    "papermill": {
     "duration": 0.004271,
     "end_time": "2024-11-12T21:40:56.432610",
     "exception": false,
     "start_time": "2024-11-12T21:40:56.428339",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Call the function to scrape the website\n",
    "\n",
    "Function arguments of **get_text_from_webpages()** are defined as:\n",
    "* root : initial webpage, to be scraped and whose links will be scraped, as well\n",
    "* parentFolder : website folder to inspect, external webpages in the site structure will be skipped\n",
    "* savePath : folder where to save .json files, containing the content of the webpages\n",
    "* numPages : maximum number of webpages to download, starting from the initial page (root argument)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a71a1545",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-12T21:40:56.444642Z",
     "iopub.status.busy": "2024-11-12T21:40:56.444031Z",
     "iopub.status.idle": "2024-11-12T21:41:30.067158Z",
     "shell.execute_reply": "2024-11-12T21:41:30.065961Z"
    },
    "papermill": {
     "duration": 33.632598,
     "end_time": "2024-11-12T21:41:30.069710",
     "exception": false,
     "start_time": "2024-11-12T21:40:56.437112",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "JSON file crawled/www.hitachienergy.com_products-and-solutions_.json created from url https://www.hitachienergy.com/products-and-solutions/\n",
      "JSON file crawled/www.hitachienergy.com_products-and-solutions_cybersecurity_alerts-and-notifications.json created from url https://www.hitachienergy.com/products-and-solutions/cybersecurity/alerts-and-notifications\n",
      "JSON file crawled/www.hitachienergy.com_products-and-solutions_cybersecurity.json created from url https://www.hitachienergy.com/products-and-solutions/cybersecurity\n",
      "JSON file crawled/www.hitachienergy.com_products-and-solutions_cybersecurity_reporting.json created from url https://www.hitachienergy.com/products-and-solutions/cybersecurity/reporting\n",
      "JSON file  created from url https://www.hitachienergy.com/products-and-solutions/cybersecurity/reporting/report-a-vulnerability-web-form\n",
      "JSON file crawled/www.hitachienergy.com_products-and-solutions_digitalization.json created from url https://www.hitachienergy.com/products-and-solutions/digitalization\n",
      "JSON file crawled/www.hitachienergy.com_products-and-solutions_grid-edge-solutions.json created from url https://www.hitachienergy.com/products-and-solutions/grid-edge-solutions\n",
      "JSON file crawled/www.hitachienergy.com_products-and-solutions_energy-portfolio-management.json created from url https://www.hitachienergy.com/products-and-solutions/energy-portfolio-management\n",
      "JSON file crawled/www.hitachienergy.com_products-and-solutions_energy-portfolio-management_enterprise_capacity-expansion.json created from url https://www.hitachienergy.com/products-and-solutions/energy-portfolio-management/enterprise/capacity-expansion\n",
      "JSON file crawled/www.hitachienergy.com_products-and-solutions_energy-portfolio-management_enterprise.json created from url https://www.hitachienergy.com/products-and-solutions/energy-portfolio-management/enterprise\n"
     ]
    }
   ],
   "source": [
    "parent_folder = \"/products-and-solutions/\"\n",
    "initial_url = \"https://www.hitachienergy.com/products-and-solutions/\"\n",
    "save_folder = \"crawled\"\n",
    "num_pages_to_download = 10\n",
    "parsed_text = get_text_from_webpages(\n",
    "    root=initial_url, parentFolder=parent_folder, savePath=save_folder,numPages=num_pages_to_download\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "038aff83",
   "metadata": {
    "papermill": {
     "duration": 0.004767,
     "end_time": "2024-11-12T21:41:30.080069",
     "exception": false,
     "start_time": "2024-11-12T21:41:30.075302",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "The variable **parsedText** contains the merged content from all the pages.\n",
    "\n",
    "To distinguish between pages and help Gemini recognizing them, two sentences are added:\n",
    "* Before the content of each webpage\n",
    "* After the content of each webpage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbcf8c6e",
   "metadata": {
    "papermill": {
     "duration": 0.005093,
     "end_time": "2024-11-12T21:41:30.090169",
     "exception": false,
     "start_time": "2024-11-12T21:41:30.085076",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Start a chat with Gemini"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5d74ed02",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-12T21:41:30.103095Z",
     "iopub.status.busy": "2024-11-12T21:41:30.102650Z",
     "iopub.status.idle": "2024-11-12T21:41:31.443934Z",
     "shell.execute_reply": "2024-11-12T21:41:31.442680Z"
    },
    "papermill": {
     "duration": 1.350434,
     "end_time": "2024-11-12T21:41:31.446620",
     "exception": false,
     "start_time": "2024-11-12T21:41:30.096186",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# API key got here: https://ai.google.dev/tutorials/setup\n",
    "\n",
    "import google.generativeai as genai\n",
    "from kaggle_secrets import UserSecretsClient\n",
    "\n",
    "user_secrets = UserSecretsClient()\n",
    "api_key = user_secrets.get_secret(\"GEMINI_API_KEY\")\n",
    "genai.configure(api_key = api_key)\n",
    "\n",
    "model = genai.GenerativeModel(model_name='gemini-1.5-pro-latest')\n",
    "\n",
    "chat = model.start_chat()\n",
    "\n",
    "# example how to send prompts\n",
    "# response = chat.send_message('Recommend me ways to fetch information from a website and forward it to you')\n",
    "# print(response.text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f093367c",
   "metadata": {
    "papermill": {
     "duration": 0.004843,
     "end_time": "2024-11-12T21:41:31.456684",
     "exception": false,
     "start_time": "2024-11-12T21:41:31.451841",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Pass the content of the website to Gemini"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a468f3c2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-12T21:41:31.468774Z",
     "iopub.status.busy": "2024-11-12T21:41:31.468220Z",
     "iopub.status.idle": "2024-11-12T21:41:31.474950Z",
     "shell.execute_reply": "2024-11-12T21:41:31.473622Z"
    },
    "papermill": {
     "duration": 0.015294,
     "end_time": "2024-11-12T21:41:31.477169",
     "exception": false,
     "start_time": "2024-11-12T21:41:31.461875",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The system prompt is:\n",
      "You are given the content of a website.\n",
      "The content for each webpage of the website is after the sentence \"The page number X with name PAGENAME has the following content: \" and before the sentence \"The page number X with name PAGENAME ends here.\" where X is the number of the page and PAGENAME is the name of the webpage.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "system_prompt = \"You are given the content of a website.\\nThe content for each webpage of the website is after the sentence \\\"The page number X with name PAGENAME has the following content: \\\" and before the sentence \\\"The page number X with name PAGENAME ends here.\\\" where X is the number of the page and PAGENAME is the name of the webpage.\\n\"\n",
    "print(f\"The system prompt is:\\n{system_prompt}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b9d7e1af",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-12T21:41:31.489558Z",
     "iopub.status.busy": "2024-11-12T21:41:31.489141Z",
     "iopub.status.idle": "2024-11-12T21:41:31.494826Z",
     "shell.execute_reply": "2024-11-12T21:41:31.493444Z"
    },
    "papermill": {
     "duration": 0.014896,
     "end_time": "2024-11-12T21:41:31.497332",
     "exception": false,
     "start_time": "2024-11-12T21:41:31.482436",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "user_prompt = \"Use the content of the website to illustrate the products of the company.\\nHighlight the possible business areas in which the product can be sold competitively.\\n\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "da65c857",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-12T21:41:31.510312Z",
     "iopub.status.busy": "2024-11-12T21:41:31.509256Z",
     "iopub.status.idle": "2024-11-12T21:41:31.514506Z",
     "shell.execute_reply": "2024-11-12T21:41:31.513423Z"
    },
    "papermill": {
     "duration": 0.014044,
     "end_time": "2024-11-12T21:41:31.516802",
     "exception": false,
     "start_time": "2024-11-12T21:41:31.502758",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "information = f\"The content of the website is:\\n{parsed_text}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f65a0d7c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-12T21:41:31.528961Z",
     "iopub.status.busy": "2024-11-12T21:41:31.528589Z",
     "iopub.status.idle": "2024-11-12T21:41:46.757437Z",
     "shell.execute_reply": "2024-11-12T21:41:46.756174Z"
    },
    "papermill": {
     "duration": 15.24437,
     "end_time": "2024-11-12T21:41:46.766477",
     "exception": false,
     "start_time": "2024-11-12T21:41:31.522107",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hitachi Energy offers a suite of products and solutions focused on empowering energy stakeholders to navigate the complex and evolving energy landscape. Here's a breakdown of their offerings and potential business areas:\n",
      "\n",
      "**Products & Solutions:**\n",
      "\n",
      "* **Energy Portfolio Management:** This encompasses software, consulting, and market intelligence to support investment decisions, operations, and trading across the energy spectrum.  This includes functionalities for forecasting, portfolio optimization, market operations, and assessing infrastructure investments (including renewables like wind and PV).\n",
      "* **Capacity Expansion:** This solution aids in long-term resource planning, capacity expansion, and emissions compliance planning.  It helps users model different scenarios to optimize resource allocation and meet regulatory requirements.\n",
      "* **Digitalization Solutions:** Hitachi Energy focuses on digitalization to manage the increasing complexity of the modern grid, including distributed energy resources, microgrids, and battery storage.  This likely involves software and services for data integration, analysis, and optimization.\n",
      "* **Grid Edge Solutions (e-mesh™):** This portfolio includes battery storage systems, automation, and digital capabilities to manage and optimize energy at the grid edge. Key applications include microgrids, renewable energy integration, and EV charging infrastructure management.\n",
      "* **Cybersecurity Solutions:**  Recognizing the increasing vulnerability of digitalized energy systems, Hitachi Energy offers cybersecurity solutions designed to protect against evolving threats. This likely includes security software, services, and consulting.\n",
      "* **Market Analytics:**  This offering provides rigorously curated energy market data and sophisticated analysis tools for forecasting, planning, asset valuation, and risk management. This supports both long-term strategic decisions and short-term operational optimization.\n",
      "* **Substation Automation, Protection & Control:** While details are limited, this suggests a traditional product line focusing on the core infrastructure of the electrical grid.\n",
      "\n",
      "**Business Areas & Competitive Advantages:**\n",
      "\n",
      "* **Utilities:**  A core customer base, benefiting from all product lines, particularly portfolio management, capacity expansion, grid edge solutions, cybersecurity, and market analytics. Hitachi Energy's focus on digitalization addresses the key challenges utilities face in modernizing their operations.\n",
      "* **Renewable Energy Developers:** The focus on renewable energy integration, grid edge solutions, and capacity expansion makes Hitachi Energy a strong partner for developers of wind, solar, and other renewable energy projects.\n",
      "* **Transmission Planners:** Solutions for capacity expansion, market analytics, and substation automation directly address the needs of transmission planners, allowing them to optimize infrastructure development and manage congestion.\n",
      "* **Market Operators:** Market analytics and portfolio management tools can assist market operators in ensuring grid stability and efficient market operation in the face of increasing complexity.\n",
      "* **Energy Traders:** The portfolio management and market analytics offerings cater to energy traders, providing the necessary data and tools for informed decision-making and risk management.\n",
      "* **Private Enterprises:** Businesses with significant energy consumption can benefit from Hitachi Energy's digitalization and grid edge solutions to optimize energy usage and reduce costs.\n",
      "* **Regulatory Agencies:** Capacity expansion and market analytics tools can support regulatory agencies in developing effective energy policies and monitoring market compliance.\n",
      "* **Mining, Metals, Oil & Gas:** Hitachi Energy explicitly mentions these sectors as targets for its digitalization solutions, focusing on optimizing operations, reducing environmental impact, and improving asset management.\n",
      "\n",
      "\n",
      "Hitachi Energy's competitive advantage lies in its comprehensive approach, combining deep domain expertise, advanced analytics, and a focus on digitalization to address the multifaceted challenges of the modern energy landscape. Their emphasis on integrating solutions across the energy value chain positions them well to serve a diverse range of customers and contribute to a more sustainable and efficient energy future.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "response = chat.send_message(system_prompt + user_prompt + information)\n",
    "print(response.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d03a82ac",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-12T21:41:46.784968Z",
     "iopub.status.busy": "2024-11-12T21:41:46.784534Z",
     "iopub.status.idle": "2024-11-12T21:41:46.790724Z",
     "shell.execute_reply": "2024-11-12T21:41:46.789640Z"
    },
    "papermill": {
     "duration": 0.018683,
     "end_time": "2024-11-12T21:41:46.793509",
     "exception": false,
     "start_time": "2024-11-12T21:41:46.774826",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token count: 3727\n"
     ]
    }
   ],
   "source": [
    "usage_metadata = response.usage_metadata\n",
    "print(f\"Token count: {usage_metadata.total_token_count}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9aa75aa",
   "metadata": {
    "papermill": {
     "duration": 0.007886,
     "end_time": "2024-11-12T21:41:46.809900",
     "exception": false,
     "start_time": "2024-11-12T21:41:46.802014",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "none",
   "dataSources": [],
   "dockerImageVersionId": 30786,
   "isGpuEnabled": false,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 78.525822,
   "end_time": "2024-11-12T21:41:47.439384",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-11-12T21:40:28.913562",
   "version": "2.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}