data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "98814909-6e95-4b4b-b18f-c5e5e616fca7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# URL of the webpage you want to scrape\n",
    "url = 'https://www.paulgraham.com/articles.html'\n",
    "\n",
    "# Fetch the content of the webpage\n",
    "response = requests.get(url)\n",
    "\n",
    "webpages= []\n",
    "\n",
    "# Check if the request was successful\n",
    "if response.status_code == 200:\n",
    "    # Parse the HTML content\n",
    "    soup = BeautifulSoup(response.text, 'html.parser')\n",
    "\n",
    "    # Find all 'a' tags\n",
    "    a_tags = soup.find_all('a')\n",
    "\n",
    "    # Extract 'href' attributes\n",
    "    hrefs = [a.get('href') for a in a_tags if a.get('href')]\n",
    "\n",
    "    # Print the extracted hrefs\n",
    "    for href in hrefs:\n",
    "        webpages.append(href)\n",
    "else:\n",
    "    print(f\"Failed to retrieve the webpage. Status code: {response.status_code}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c3eb7b97-6576-4bef-9cd6-50cb462b83d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "webpages = [\n",
    "    \"index.html\",\n",
    "    \"wisdom.html\",\n",
    "    \"kids.html\",\n",
    "    \"selfindulgence.html\",\n",
    "    \"reddits.html\",\n",
    "    \"google.html\",\n",
    "    \"best.html\",\n",
    "    \"superlinear.html\",\n",
    "    \"greatwork.html\",\n",
    "    \"getideas.html\",\n",
    "    \"read.html\",\n",
    "    \"want.html\",\n",
    "    \"alien.html\",\n",
    "    \"users.html\",\n",
    "    \"heresy.html\",\n",
    "    \"words.html\",\n",
    "    \"goodtaste.html\",\n",
    "    \"smart.html\",\n",
    "    \"weird.html\",\n",
    "    \"hwh.html\",\n",
    "    \"own.html\",\n",
    "    \"fn.html\",\n",
    "    \"newideas.html\",\n",
    "    \"nft.html\",\n",
    "    \"real.html\",\n",
    "    \"richnow.html\",\n",
    "    \"simply.html\",\n",
    "    \"donate.html\",\n",
    "    \"worked.html\",\n",
    "    \"earnest.html\",\n",
    "    \"ace.html\",\n",
    "    \"airbnbs.html\",\n",
    "    \"think.html\",\n",
    "    \"early.html\",\n",
    "    \"wtax.html\",\n",
    "    \"conformism.html\",\n",
    "    \"orth.html\",\n",
    "    \"cred.html\",\n",
    "    \"useful.html\",\n",
    "    \"noob.html\",\n",
    "    \"fh.html\",\n",
    "    \"mod.html\",\n",
    "    \"fp.html\",\n",
    "    \"kids.html\",\n",
    "    \"lesson.html\",\n",
    "    \"nov.html\",\n",
    "    \"genius.html\",\n",
    "    \"sun.html\",\n",
    "    \"pow.html\",\n",
    "    \"disc.html\",\n",
    "    \"pgh.html\",\n",
    "    \"vb.html\",\n",
    "    \"ineq.html\",\n",
    "    \"re.html\",\n",
    "    \"jessica.html\",\n",
    "    \"bias.html\",\n",
    "    \"talk.html\",\n",
    "    \"aord.html\",\n",
    "    \"safe.html\",\n",
    "    \"name.html\",\n",
    "    \"altair.html\",\n",
    "    \"ronco.html\",\n",
    "    \"work.html\",\n",
    "    \"corpdev.html\",\n",
    "    \"95.html\",\n",
    "    \"ecw.html\",\n",
    "    \"know.html\",\n",
    "    \"pinch.html\",\n",
    "    \"mean.html\",\n",
    "    \"before.html\",\n",
    "    \"fr.html\",\n",
    "    \"herd.html\",\n",
    "    \"convince.html\",\n",
    "    \"ds.html\",\n",
    "    \"invtrend.html\",\n",
    "    \"startupideas.html\",\n",
    "    \"hw.html\",\n",
    "    \"growth.html\",\n",
    "    \"swan.html\",\n",
    "    \"todo.html\",\n",
    "    \"speak.html\",\n",
    "    \"ycstart.html\",\n",
    "    \"property.html\",\n",
    "    \"ambitious.html\",\n",
    "    \"word.html\",\n",
    "    \"schlep.html\",\n",
    "    \"vw.html\",\n",
    "    \"hubs.html\",\n",
    "    \"patentpledge.html\",\n",
    "    \"airbnb.html\",\n",
    "    \"control.html\",\n",
    "    \"tablets.html\",\n",
    "    \"founders.html\",\n",
    "    \"superangels.html\",\n",
    "    \"seesv.html\",\n",
    "    \"hiresfund.html\",\n",
    "    \"yahoo.html\",\n",
    "    \"future.html\",\n",
    "    \"addiction.html\",\n",
    "    \"top.html\",\n",
    "    \"selfindulgence.html\",\n",
    "    \"organic.html\",\n",
    "    \"apple.html\",\n",
    "    \"really.html\",\n",
    "    \"discover.html\",\n",
    "    \"publishing.html\",\n",
    "    \"nthings.html\",\n",
    "    \"determination.html\",\n",
    "    \"kate.html\",\n",
    "    \"segway.html\",\n",
    "    \"ramenprofitable.html\",\n",
    "    \"makersschedule.html\",\n",
    "    \"revolution.html\",\n",
    "    \"twitter.html\",\n",
    "    \"foundervisa.html\",\n",
    "    \"5founders.html\",\n",
    "    \"relres.html\",\n",
    "    \"angelinvesting.html\",\n",
    "    \"convergence.html\",\n",
    "    \"maybe.html\",\n",
    "    \"hackernews.html\",\n",
    "    \"13sentences.html\",\n",
    "    \"identity.html\",\n",
    "    \"credentials.html\",\n",
    "    \"divergence.html\",\n",
    "    \"highres.html\",\n",
    "    \"artistsship.html\",\n",
    "    \"badeconomy.html\",\n",
    "    \"fundraising.html\",\n",
    "    \"prcmc.html\",\n",
    "    \"cities.html\",\n",
    "    \"distraction.html\",\n",
    "    \"lies.html\",\n",
    "    \"good.html\",\n",
    "    \"googles.html\",\n",
    "    \"heroes.html\",\n",
    "    \"disagree.html\",\n",
    "    \"boss.html\",\n",
    "    \"ycombinator.html\",\n",
    "    \"trolls.html\",\n",
    "    \"newthings.html\",\n",
    "    \"startuphubs.html\",\n",
    "    \"webstartups.html\",\n",
    "    \"philosophy.html\",\n",
    "    \"colleges.html\",\n",
    "    \"die.html\",\n",
    "    \"head.html\",\n",
    "    \"stuff.html\",\n",
    "    \"equity.html\",\n",
    "    \"unions.html\",\n",
    "    \"guidetoinvestors.html\",\n",
    "    \"judgement.html\",\n",
    "    \"microsoft.html\",\n",
    "    \"notnot.html\",\n",
    "    \"wisdom.html\",\n",
    "    \"foundersatwork.html\",\n",
    "    \"goodart.html\",\n",
    "    \"startupmistakes.html\",\n",
    "    \"mit.html\",\n",
    "    \"investors.html\",\n",
    "    \"copy.html\",\n",
    "    \"island.html\",\n",
    "    \"marginal.html\",\n",
    "    \"america.html\",\n",
    "    \"siliconvalley.html\",\n",
    "    \"startuplessons.html\",\n",
    "    \"randomness.html\",\n",
    "    \"softwarepatents.html\",\n",
    "    \"6631327.html\",\n",
    "    \"whyyc.html\",\n",
    "    \"love.html\",\n",
    "    \"procrastination.html\",\n",
    "    \"web20.html\",\n",
    "    \"startupfunding.html\",\n",
    "    \"vcsqueeze.html\",\n",
    "    \"ideas.html\",\n",
    "    \"sfp.html\",\n",
    "    \"inequality.html\",\n",
    "    \"ladder.html\",\n",
    "    \"opensource.html\",\n",
    "    \"hiring.html\",\n",
    "    \"submarine.html\",\n",
    "    \"bronze.html\",\n",
    "    \"mac.html\",\n",
    "    \"writing44.html\",\n",
    "    \"college.html\",\n",
    "    \"venturecapital.html\",\n",
    "    \"start.html\",\n",
    "    \"hs.html\",\n",
    "    \"usa.html\",\n",
    "    \"charisma.html\",\n",
    "    \"polls.html\",\n",
    "    \"laundry.html\",\n",
    "    \"bubble.html\",\n",
    "    \"essay.html\",\n",
    "    \"pypar.html\",\n",
    "    \"gh.html\",\n",
    "    \"gap.html\",\n",
    "    \"wealth.html\",\n",
    "    \"gba.html\",\n",
    "    \"say.html\",\n",
    "    \"ffb.html\",\n",
    "    \"hp.html\",\n",
    "    \"iflisp.html\",\n",
    "    \"hundred.html\",\n",
    "    \"nerds.html\",\n",
    "    \"better.html\",\n",
    "    \"desres.html\",\n",
    "    \"spam.html\",\n",
    "    \"icad.html\",\n",
    "    \"power.html\",\n",
    "    \"fix.html\",\n",
    "    \"taste.html\",\n",
    "    \"noop.html\",\n",
    "    \"diff.html\",\n",
    "    \"road.html\",\n",
    "    \"rootsoflisp.html\",\n",
    "    \"langdes.html\",\n",
    "    \"popular.html\",\n",
    "    \"javacover.html\",\n",
    "    \"avg.html\",\n",
    "    \"lwba.html\",\n",
    "    \"progbot.html\",\n",
    "    \"prop62.html\"\n",
    "]\n",
    "\n",
    "# Remove duplicates by converting the list to a set and then back to a list\n",
    "webpages = list(set(webpages))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ce8cf219-f1aa-4c58-a1fe-b67535b168b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_new_line(text):\n",
    "    split_parts = text.split(\"\\n\")\n",
    "    join_substring = \" \"\n",
    "    joined_string = join_substring.join(split_parts)\n",
    "    return joined_string\n",
    "\n",
    "\n",
    "\n",
    "def get_first_table_text(url):\n",
    "    \"\"\"\n",
    "    Fetches the text content of the first table found on the given webpage.\n",
    "\n",
    "    Args:\n",
    "    url (str): The URL of the webpage to scrape.\n",
    "\n",
    "    Returns:\n",
    "    str: The text content of the first table, or a message if no table is found or an error occurs.\n",
    "    \"\"\"\n",
    "\n",
    "    try:\n",
    "        # Fetch the content of the webpage\n",
    "        response = requests.get(url)\n",
    "\n",
    "        # Check if the request was successful\n",
    "        if response.status_code == 200:\n",
    "            # Parse the HTML content\n",
    "            soup = BeautifulSoup(response.text, 'html.parser')\n",
    "\n",
    "            # Find all tables\n",
    "            tables = soup.find_all('table')\n",
    "\n",
    "            if tables:\n",
    "                # Get the first table\n",
    "                first_table = tables[0]\n",
    "\n",
    "                # Extract text content from the first table\n",
    "                raw_text = first_table.get_text()\n",
    "                cleaned_text = raw_text.replace('\\n', ' ').replace('\\\\', '')\n",
    "\n",
    "                return cleaned_text.strip()\n",
    "            else:\n",
    "                return \"No tables found on the webpage.\"\n",
    "        else:\n",
    "            return f\"Failed to retrieve the webpage. Status code: {response.status_code}\"\n",
    "    except Exception as e:\n",
    "        return f\"An error occurred: {e}\"\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "963c42c6-ba8e-4075-ad35-286d17aecbd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import openai\n",
    "import numpy as np\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "embedding_dimensions = 512\n",
    "\n",
    "from openai import OpenAI\n",
    "import numpy as np\n",
    "\n",
    "client = OpenAI()\n",
    "\n",
    "def normalize_l2(x):\n",
    "    x = np.array(x)\n",
    "    if x.ndim == 1:\n",
    "        norm = np.linalg.norm(x)\n",
    "        if norm == 0:\n",
    "            return x\n",
    "        return x / norm\n",
    "    else:\n",
    "        norm = np.linalg.norm(x, 2, axis=1, keepdims=True)\n",
    "        return np.where(norm == 0, x, x / norm)\n",
    "\n",
    "def convert_to_vector(texts):\n",
    "\n",
    "    # Try to generate embeddings for a list of texts using a pre-trained model and handle any exceptions.\n",
    "    try:\n",
    "        response = openai.embeddings.create(\n",
    "            input=texts,\n",
    "            model=\"text-embedding-3-small\"\n",
    "        )\n",
    "        embeddings = np.array([item.embedding for item in response.data])\n",
    "        return embeddings\n",
    "    except Exception as e:\n",
    "        print(\"An error occurred:\", e)\n",
    "        return np.array([])  # Return an empty array in case of an error\n",
    "\n",
    "def chunk_text(text):\n",
    "    # Split the input text into individual sentences.\n",
    "    single_sentences_list = _split_sentences(text)\n",
    "    # Combine adjacent sentences to form a context window around each sentence.\n",
    "    combined_sentences = _combine_sentences(single_sentences_list)\n",
    "    \n",
    "    # Convert the combined sentences into vector representations using a neural network model.\n",
    "    embeddings = convert_to_vector(combined_sentences)\n",
    "    \n",
    "    # Calculate the cosine distances between consecutive combined sentence embeddings to measure similarity.\n",
    "    distances = _calculate_cosine_distances(embeddings)\n",
    "    \n",
    "    # Determine the threshold distance for identifying breakpoints based on the 80th percentile of all distances.\n",
    "    breakpoint_percentile_threshold = 80\n",
    "    breakpoint_distance_threshold = np.percentile(distances, breakpoint_percentile_threshold)\n",
    "    # Find all indices where the distance exceeds the calculated threshold, indicating a potential chunk breakpoint.\n",
    "    indices_above_thresh = [i for i, distance in enumerate(distances) if distance > breakpoint_distance_threshold]\n",
    "    # Initialize the list of chunks and a variable to track the start of the next chunk.\n",
    "    chunks = []\n",
    "    start_index = 0\n",
    "    # Loop through the identified breakpoints and create chunks accordingly.\n",
    "    for index in indices_above_thresh:\n",
    "        chunk = ' '.join(single_sentences_list[start_index:index+1])\n",
    "        chunks.append(chunk)\n",
    "        start_index = index + 1\n",
    "    \n",
    "    # If there are any sentences left after the last breakpoint, add them as the final chunk.\n",
    "    if start_index < len(single_sentences_list):\n",
    "        chunk = ' '.join(single_sentences_list[start_index:])\n",
    "        chunks.append(chunk)\n",
    "    \n",
    "    # Return the list of text chunks.\n",
    "    return chunks\n",
    "\n",
    "def _split_sentences(text):\n",
    "    # Use regular expressions to split the text into sentences based on punctuation followed by whitespace.\n",
    "    sentences = re.split(r'(?<=[.?!])\\s+', text)\n",
    "    return sentences\n",
    "\n",
    "def _combine_sentences(sentences):\n",
    "    # Create a buffer by combining each sentence with its previous and next sentence to provide a wider context.\n",
    "    combined_sentences = []\n",
    "    for i in range(len(sentences)):\n",
    "        combined_sentence = sentences[i]\n",
    "        if i > 0:\n",
    "            combined_sentence = sentences[i-1] + ' ' + combined_sentence\n",
    "        if i < len(sentences) - 1:\n",
    "            combined_sentence += ' ' + sentences[i+1]\n",
    "        combined_sentences.append(combined_sentence)\n",
    "    return combined_sentences\n",
    "\n",
    "def _calculate_cosine_distances(embeddings):\n",
    "    # Calculate the cosine distance (1 - cosine similarity) between consecutive embeddings.\n",
    "    distances = []\n",
    "    for i in range(len(embeddings) - 1):\n",
    "        similarity = cosine_similarity([embeddings[i]], [embeddings[i + 1]])[0][0]\n",
    "        distance = 1 - similarity\n",
    "        distances.append(distance)\n",
    "    return distances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "595cc89a-6d69-4cb8-8ef2-5a40304023c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pinecone import Pinecone, ServerlessSpec\n",
    "\n",
    "pc = Pinecone(api_key=pinecone_api_key)\n",
    "\n",
    "pc.create_index(\n",
    "  name=\"pg-blogs\",\n",
    "  dimension=1536,\n",
    "  metric=\"cosine\",\n",
    "  spec=ServerlessSpec(\n",
    "    cloud=\"aws\",\n",
    "    region=\"us-east-1\"\n",
    "  )\n",
    ")\n",
    "\n",
    "index = pc.Index(\"pg-blogs\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "4e845991-f5f3-4d36-bc5b-d37b9ce649c8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting 1 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "1 of 221 COMPLETED\n",
      "Starting 2 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "2 of 221 COMPLETED\n",
      "Starting 3 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "3 of 221 COMPLETED\n",
      "Starting 4 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "4 of 221 COMPLETED\n",
      "Starting 5 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "5 of 221 COMPLETED\n",
      "Starting 6 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "6 of 221 COMPLETED\n",
      "Starting 7 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "7 of 221 COMPLETED\n",
      "Starting 8 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "8 of 221 COMPLETED\n",
      "Starting 9 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "9 of 221 COMPLETED\n",
      "Starting 10 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "10 of 221 COMPLETED\n",
      "Starting 11 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "11 of 221 COMPLETED\n",
      "Starting 12 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "12 of 221 COMPLETED\n",
      "Starting 13 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "13 of 221 COMPLETED\n",
      "Starting 14 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "14 of 221 COMPLETED\n",
      "Starting 15 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "15 of 221 COMPLETED\n",
      "Starting 16 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "16 of 221 COMPLETED\n",
      "Starting 17 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "17 of 221 COMPLETED\n",
      "Starting 18 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "18 of 221 COMPLETED\n",
      "Starting 19 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "19 of 221 COMPLETED\n",
      "Starting 20 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "20 of 221 COMPLETED\n",
      "Starting 21 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "21 of 221 COMPLETED\n",
      "Starting 22 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "22 of 221 COMPLETED\n",
      "Starting 23 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "23 of 221 COMPLETED\n",
      "Starting 24 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "24 of 221 COMPLETED\n",
      "Starting 25 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "25 of 221 COMPLETED\n",
      "Starting 26 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "26 of 221 COMPLETED\n",
      "Starting 27 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "27 of 221 COMPLETED\n",
      "Starting 28 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "28 of 221 COMPLETED\n",
      "Starting 29 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "29 of 221 COMPLETED\n",
      "Starting 30 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "30 of 221 COMPLETED\n",
      "Starting 31 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "31 of 221 COMPLETED\n",
      "Starting 32 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "32 of 221 COMPLETED\n",
      "Starting 33 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "33 of 221 COMPLETED\n",
      "Starting 34 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "34 of 221 COMPLETED\n",
      "Starting 35 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "35 of 221 COMPLETED\n",
      "Starting 36 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "36 of 221 COMPLETED\n",
      "Starting 37 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "37 of 221 COMPLETED\n",
      "Starting 38 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "38 of 221 COMPLETED\n",
      "Starting 39 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "39 of 221 COMPLETED\n",
      "Starting 40 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "40 of 221 COMPLETED\n",
      "Starting 41 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "41 of 221 COMPLETED\n",
      "Starting 42 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "42 of 221 COMPLETED\n",
      "Starting 43 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "43 of 221 COMPLETED\n",
      "Starting 44 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "44 of 221 COMPLETED\n",
      "Starting 45 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "45 of 221 COMPLETED\n",
      "Starting 46 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "46 of 221 COMPLETED\n",
      "Starting 47 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "47 of 221 COMPLETED\n",
      "Starting 48 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "48 of 221 COMPLETED\n",
      "Starting 49 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "49 of 221 COMPLETED\n",
      "Starting 50 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "50 of 221 COMPLETED\n",
      "Starting 51 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "51 of 221 COMPLETED\n",
      "Starting 52 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "52 of 221 COMPLETED\n",
      "Starting 53 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "53 of 221 COMPLETED\n",
      "Starting 54 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "54 of 221 COMPLETED\n",
      "Starting 55 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "55 of 221 COMPLETED\n",
      "Starting 56 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "56 of 221 COMPLETED\n",
      "Starting 57 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "57 of 221 COMPLETED\n",
      "Starting 58 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "58 of 221 COMPLETED\n",
      "Starting 59 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "59 of 221 COMPLETED\n",
      "Starting 60 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "60 of 221 COMPLETED\n",
      "Starting 61 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "61 of 221 COMPLETED\n",
      "Starting 62 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "62 of 221 COMPLETED\n",
      "Starting 63 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "63 of 221 COMPLETED\n",
      "Starting 64 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "64 of 221 COMPLETED\n",
      "Starting 65 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "65 of 221 COMPLETED\n",
      "Starting 66 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "66 of 221 COMPLETED\n",
      "Starting 67 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "67 of 221 COMPLETED\n",
      "Starting 68 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "68 of 221 COMPLETED\n",
      "Starting 69 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "69 of 221 COMPLETED\n",
      "Starting 70 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "70 of 221 COMPLETED\n",
      "Starting 71 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "71 of 221 COMPLETED\n",
      "Starting 72 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "72 of 221 COMPLETED\n",
      "Starting 73 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "73 of 221 COMPLETED\n",
      "Starting 74 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "74 of 221 COMPLETED\n",
      "Starting 75 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "75 of 221 COMPLETED\n",
      "Starting 76 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "76 of 221 COMPLETED\n",
      "Starting 77 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "77 of 221 COMPLETED\n",
      "Starting 78 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "78 of 221 COMPLETED\n",
      "Starting 79 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "79 of 221 COMPLETED\n",
      "Starting 80 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "80 of 221 COMPLETED\n",
      "Starting 81 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "81 of 221 COMPLETED\n",
      "Starting 82 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "82 of 221 COMPLETED\n",
      "Starting 83 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "83 of 221 COMPLETED\n",
      "Starting 84 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "84 of 221 COMPLETED\n",
      "Starting 85 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "85 of 221 COMPLETED\n",
      "Starting 86 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "86 of 221 COMPLETED\n",
      "Starting 87 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "87 of 221 COMPLETED\n",
      "Starting 88 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "88 of 221 COMPLETED\n",
      "Starting 89 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "89 of 221 COMPLETED\n",
      "Starting 90 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "90 of 221 COMPLETED\n",
      "Starting 91 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "91 of 221 COMPLETED\n",
      "Starting 92 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "92 of 221 COMPLETED\n",
      "Starting 93 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "93 of 221 COMPLETED\n",
      "Starting 94 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "94 of 221 COMPLETED\n",
      "Starting 95 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "95 of 221 COMPLETED\n",
      "Starting 96 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "96 of 221 COMPLETED\n",
      "Starting 97 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "97 of 221 COMPLETED\n",
      "Starting 98 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "98 of 221 COMPLETED\n",
      "Starting 99 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "99 of 221 COMPLETED\n",
      "Starting 100 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "100 of 221 COMPLETED\n",
      "Starting 101 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "101 of 221 COMPLETED\n",
      "Starting 102 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "102 of 221 COMPLETED\n",
      "Starting 103 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "103 of 221 COMPLETED\n",
      "Starting 104 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "104 of 221 COMPLETED\n",
      "Starting 105 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "105 of 221 COMPLETED\n",
      "Starting 106 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "106 of 221 COMPLETED\n",
      "Starting 107 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "107 of 221 COMPLETED\n",
      "Starting 108 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "108 of 221 COMPLETED\n",
      "Starting 109 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "109 of 221 COMPLETED\n",
      "Starting 110 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "110 of 221 COMPLETED\n",
      "Starting 111 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "111 of 221 COMPLETED\n",
      "Starting 112 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "112 of 221 COMPLETED\n",
      "Starting 113 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "113 of 221 COMPLETED\n",
      "Starting 114 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "114 of 221 COMPLETED\n",
      "Starting 115 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "115 of 221 COMPLETED\n",
      "Starting 116 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "116 of 221 COMPLETED\n",
      "Starting 117 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "117 of 221 COMPLETED\n",
      "Starting 118 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "118 of 221 COMPLETED\n",
      "Starting 119 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "119 of 221 COMPLETED\n",
      "Starting 120 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "120 of 221 COMPLETED\n",
      "Starting 121 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "121 of 221 COMPLETED\n",
      "Starting 122 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "122 of 221 COMPLETED\n",
      "Starting 123 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "123 of 221 COMPLETED\n",
      "Starting 124 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "124 of 221 COMPLETED\n",
      "Starting 125 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "125 of 221 COMPLETED\n",
      "Starting 126 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "126 of 221 COMPLETED\n",
      "Starting 127 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "127 of 221 COMPLETED\n",
      "Starting 128 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "128 of 221 COMPLETED\n",
      "Starting 129 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "129 of 221 COMPLETED\n",
      "Starting 130 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "130 of 221 COMPLETED\n",
      "Starting 131 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "131 of 221 COMPLETED\n",
      "Starting 132 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "132 of 221 COMPLETED\n",
      "Starting 133 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "133 of 221 COMPLETED\n",
      "Starting 134 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "134 of 221 COMPLETED\n",
      "Starting 135 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "135 of 221 COMPLETED\n",
      "Starting 136 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "136 of 221 COMPLETED\n",
      "Starting 137 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "137 of 221 COMPLETED\n",
      "Starting 138 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "138 of 221 COMPLETED\n",
      "Starting 139 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "139 of 221 COMPLETED\n",
      "Starting 140 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "140 of 221 COMPLETED\n",
      "Starting 141 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "141 of 221 COMPLETED\n",
      "Starting 142 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "142 of 221 COMPLETED\n",
      "Starting 143 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "143 of 221 COMPLETED\n",
      "Starting 144 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "144 of 221 COMPLETED\n",
      "Starting 145 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "145 of 221 COMPLETED\n",
      "Starting 146 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "146 of 221 COMPLETED\n",
      "Starting 147 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "147 of 221 COMPLETED\n",
      "Starting 148 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "148 of 221 COMPLETED\n",
      "Starting 149 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "149 of 221 COMPLETED\n",
      "Starting 150 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "150 of 221 COMPLETED\n",
      "Starting 151 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "151 of 221 COMPLETED\n",
      "Starting 152 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "152 of 221 COMPLETED\n",
      "Starting 153 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "153 of 221 COMPLETED\n",
      "Starting 154 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "154 of 221 COMPLETED\n",
      "Starting 155 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "155 of 221 COMPLETED\n",
      "Starting 156 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "156 of 221 COMPLETED\n",
      "Starting 157 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "157 of 221 COMPLETED\n",
      "Starting 158 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "158 of 221 COMPLETED\n",
      "Starting 159 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "159 of 221 COMPLETED\n",
      "Starting 160 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "160 of 221 COMPLETED\n",
      "Starting 161 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "161 of 221 COMPLETED\n",
      "Starting 162 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "162 of 221 COMPLETED\n",
      "Starting 163 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "163 of 221 COMPLETED\n",
      "Starting 164 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "164 of 221 COMPLETED\n",
      "Starting 165 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "165 of 221 COMPLETED\n",
      "Starting 166 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "166 of 221 COMPLETED\n",
      "Starting 167 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "167 of 221 COMPLETED\n",
      "Starting 168 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "168 of 221 COMPLETED\n",
      "Starting 169 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "169 of 221 COMPLETED\n",
      "Starting 170 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "170 of 221 COMPLETED\n",
      "Starting 171 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "171 of 221 COMPLETED\n",
      "Starting 172 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "172 of 221 COMPLETED\n",
      "Starting 173 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "173 of 221 COMPLETED\n",
      "Starting 174 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "174 of 221 COMPLETED\n",
      "Starting 175 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "175 of 221 COMPLETED\n",
      "Starting 176 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "176 of 221 COMPLETED\n",
      "Starting 177 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "177 of 221 COMPLETED\n",
      "Starting 178 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "178 of 221 COMPLETED\n",
      "Starting 179 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "179 of 221 COMPLETED\n",
      "Starting 180 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "180 of 221 COMPLETED\n",
      "Starting 181 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "181 of 221 COMPLETED\n",
      "Starting 182 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "182 of 221 COMPLETED\n",
      "Starting 183 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "183 of 221 COMPLETED\n",
      "Starting 184 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "184 of 221 COMPLETED\n",
      "Starting 185 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "185 of 221 COMPLETED\n",
      "Starting 186 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "186 of 221 COMPLETED\n",
      "Starting 187 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "187 of 221 COMPLETED\n",
      "Starting 188 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "188 of 221 COMPLETED\n",
      "Starting 189 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "189 of 221 COMPLETED\n",
      "Starting 190 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "190 of 221 COMPLETED\n",
      "Starting 191 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "191 of 221 COMPLETED\n",
      "Starting 192 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "192 of 221 COMPLETED\n",
      "Starting 193 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "193 of 221 COMPLETED\n",
      "Starting 194 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "194 of 221 COMPLETED\n",
      "Starting 195 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "195 of 221 COMPLETED\n",
      "Starting 196 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "196 of 221 COMPLETED\n",
      "Starting 197 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "197 of 221 COMPLETED\n",
      "Starting 198 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "198 of 221 COMPLETED\n",
      "Starting 199 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "199 of 221 COMPLETED\n",
      "Starting 200 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "200 of 221 COMPLETED\n",
      "Starting 201 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "201 of 221 COMPLETED\n",
      "Starting 202 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "202 of 221 COMPLETED\n",
      "Starting 203 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "203 of 221 COMPLETED\n",
      "Starting 204 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "204 of 221 COMPLETED\n",
      "Starting 205 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "205 of 221 COMPLETED\n",
      "Starting 206 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "206 of 221 COMPLETED\n",
      "Starting 207 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "207 of 221 COMPLETED\n",
      "Starting 208 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "208 of 221 COMPLETED\n",
      "Starting 209 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "209 of 221 COMPLETED\n",
      "Starting 210 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "210 of 221 COMPLETED\n",
      "Starting 211 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "211 of 221 COMPLETED\n",
      "Starting 212 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "212 of 221 COMPLETED\n",
      "Starting 213 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "213 of 221 COMPLETED\n",
      "Starting 214 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "214 of 221 COMPLETED\n",
      "Starting 215 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "215 of 221 COMPLETED\n",
      "Starting 216 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "216 of 221 COMPLETED\n",
      "Starting 217 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "217 of 221 COMPLETED\n",
      "Starting 218 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "218 of 221 COMPLETED\n",
      "Starting 219 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "219 of 221 COMPLETED\n",
      "Starting 220 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "220 of 221 COMPLETED\n",
      "Starting 221 of 221\n",
      "Text fetched from blog\n",
      "Chunks prepared from the text\n",
      "221 of 221 COMPLETED\n"
     ]
    }
   ],
   "source": [
    "import uuid\n",
    "\n",
    "i = 0\n",
    "for wp in webpages:\n",
    "    print(f\"Starting {i+1} of {len(webpages)}\")\n",
    "    text = get_first_table_text(f'https://www.paulgraham.com/{wp}')  # Correctly iterate over each webpage\n",
    "    print('Text fetched from blog')\n",
    "    chunks = chunk_text(text)\n",
    "    print('Chunks prepared from the text')\n",
    "    \n",
    "    vectors = []\n",
    "    for chunk in chunks:\n",
    "        values = convert_to_vector([chunk])[0]\n",
    "        vector = {\n",
    "            \"id\": str(uuid.uuid4()), \n",
    "            \"values\": values, \n",
    "            \"metadata\": {\"blog\": wp, \"text\": chunk}\n",
    "        }\n",
    "        vectors.append(vector)\n",
    "        \n",
    "    index.upsert(vectors=vectors, async_req=True)\n",
    "    print(f\"{i+1} of {len(webpages)} COMPLETED\")\n",
    "    i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "6fe7b8be-e182-4627-bd0e-1bb66876d2e9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hmm, that's a great question. Getting startup ideas can be quite a challenge, but it doesn't always have to be this big, grand concept that changes the world. Sometimes, the best ideas come from simple observations in our daily lives. Like, let's say you find yourself constantly frustrated with a certain task or you wish there was a better solution to a problem you face regularly – that right there could be the start of a great idea for a startup. It's all about identifying those pain points and thinking about how you can address them in a unique and effective way. And hey, if you're really stuck, it never hurts to ask yourself: what do you wish someone would make for you? Who knows, that could lead you to your next big idea.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "query = \"how to get startup ideas?\"\n",
    "\n",
    "client = OpenAI()\n",
    "\n",
    "def get_embedding(text, model=\"text-embedding-3-small\"):\n",
    "   text = text.replace(\"\\n\", \" \")\n",
    "   return client.embeddings.create(input = [text], model=model).data[0].embedding\n",
    "\n",
    "embed = get_embedding(query)\n",
    "\n",
    "res = index.query(vector=embed, top_k=10, include_metadata=True)\n",
    "text = [r['metadata']['text'] for r in res['matches']]\n",
    "text = ' '.join(text)\n",
    "\n",
    "response = client.chat.completions.create(\n",
    "    model=\"gpt-3.5-turbo\",\n",
    "    messages=[\n",
    "        {\"role\": \"system\", \"content\": \"You are paul graham and a friendly person. You will receive a question separated by &&&& and some text separated by #### from blog of paul graham to derive your answer. You need to actlike the text provided in #### are true and you can trust on it as it is from your blog only. Please respond in a natural, conversational manner, using casual expressions, pauses, and fillers like 'hmm,' 'uh,' and 'huh' to sound more human-like.\"},\n",
    "        {\"role\": \"user\", \"content\": f\"&&&&{query}&&&&####{text}####\"}\n",
    "    ]\n",
    ")\n",
    "\n",
    "# Extract the actual answer from the response\n",
    "# answer = response['choices'][0]['message']['content']\n",
    "\n",
    "print(response.choices[0].message.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2419578-945a-4509-a9de-7451c702b7f5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}