llamaIndex+mongoDB.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyOEI2RoT5jTq5m+rZ8JU6Rx",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/prakul/MongoDB-AI-Resources/blob/main/llamaIndex%2BmongoDB.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pymongo\n",
        "!pip install llama_index pypdf\n",
        "!pip3 install python-dotenv\n",
        "!pip install openai"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "B4Pqel7ONrMt",
        "outputId": "33aee6d7-bcd2-4dbe-88a1-5a3465c56170"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: pymongo in /usr/local/lib/python3.10/dist-packages (4.4.1)\n",
            "Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in /usr/local/lib/python3.10/dist-packages (from pymongo) (2.4.2)\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 3108, in _dep_map\n",
            "    return self.__dep_map\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 2901, in __getattr__\n",
            "    raise AttributeError(attr)\n",
            "AttributeError: _DistInfoDistribution__dep_map\n",
            "\n",
            "During handling of the above exception, another exception occurred:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/cli/base_command.py\", line 169, in exc_logging_wrapper\n",
            "    status = run_func(*args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/cli/req_command.py\", line 242, in wrapper\n",
            "    return func(self, options, args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/commands/install.py\", line 441, in run\n",
            "    conflicts = self._determine_conflicts(to_install)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/commands/install.py\", line 572, in _determine_conflicts\n",
            "    return check_install_conflicts(to_install)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/operations/check.py\", line 101, in check_install_conflicts\n",
            "    package_set, _ = create_package_set_from_installed()\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/operations/check.py\", line 42, in create_package_set_from_installed\n",
            "    dependencies = list(dist.iter_dependencies())\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/metadata/pkg_resources.py\", line 216, in iter_dependencies\n",
            "    return self._dist.requires(extras)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 2821, in requires\n",
            "    dm = self._dep_map\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 3110, in _dep_map\n",
            "    self.__dep_map = self._compute_dependencies()\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 3120, in _compute_dependencies\n",
            "    reqs.extend(parse_requirements(req))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 3173, in __init__\n",
            "    super(Requirement, self).__init__(requirement_string)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/packaging/requirements.py\", line 122, in __init__\n",
            "    self.specifier: SpecifierSet = SpecifierSet(req.specifier)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/packaging/specifiers.py\", line 634, in __init__\n",
            "    parsed.add(Specifier(specifier))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/packaging/specifiers.py\", line 95, in __init__\n",
            "    def __init__(self, spec: str = \"\", prereleases: Optional[bool] = None) -> None:\n",
            "KeyboardInterrupt\n",
            "\n",
            "During handling of the above exception, another exception occurred:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/bin/pip3\", line 8, in <module>\n",
            "    sys.exit(main())\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/cli/main.py\", line 79, in main\n",
            "    return command.main(cmd_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/cli/base_command.py\", line 101, in main\n",
            "    return self._main(args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/cli/base_command.py\", line 223, in _main\n",
            "    return run(options, args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/cli/base_command.py\", line 206, in exc_logging_wrapper\n",
            "    logger.critical(\"Operation cancelled by user\")\n",
            "  File \"/usr/lib/python3.10/logging/__init__.py\", line 1523, in critical\n",
            "    if self.isEnabledFor(CRITICAL):\n",
            "  File \"/usr/lib/python3.10/logging/__init__.py\", line 1724, in isEnabledFor\n",
            "    def isEnabledFor(self, level):\n",
            "KeyboardInterrupt\n",
            "^C\n",
            "Requirement already satisfied: llama_index in /usr/local/lib/python3.10/dist-packages (0.8.2.post1)\n",
            "Requirement already satisfied: pypdf in /usr/local/lib/python3.10/dist-packages (3.15.1)\n",
            "Requirement already satisfied: llama-cpp-python in /usr/local/lib/python3.10/dist-packages (0.1.77)\n",
            "Requirement already satisfied: sentence_transformers in /usr/local/lib/python3.10/dist-packages (2.2.2)\n",
            "Requirement already satisfied: tiktoken in /usr/local/lib/python3.10/dist-packages (from llama_index) (0.4.0)\n",
            "Requirement already satisfied: dataclasses-json in /usr/local/lib/python3.10/dist-packages (from llama_index) (0.5.14)\n",
            "Requirement already satisfied: langchain>=0.0.262 in /usr/local/lib/python3.10/dist-packages (from llama_index) (0.0.265)\n",
            "Requirement already satisfied: sqlalchemy>=2.0.15 in /usr/local/lib/python3.10/dist-packages (from llama_index) (2.0.19)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from llama_index) (1.23.5)\n",
            "Requirement already satisfied: tenacity<9.0.0,>=8.2.0 in /usr/local/lib/python3.10/dist-packages (from llama_index) (8.2.2)\n",
            "Requirement already satisfied: openai>=0.26.4 in /usr/local/lib/python3.10/dist-packages (from llama_index) (0.27.8)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from llama_index) (1.5.3)\n",
            "Requirement already satisfied: urllib3<2 in /usr/local/lib/python3.10/dist-packages (from llama_index) (1.26.16)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from llama_index) (2023.6.0)\n",
            "Requirement already satisfied: typing-inspect>=0.8.0 in /usr/local/lib/python3.10/dist-packages (from llama_index) (0.9.0)\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from llama_index) (4.7.1)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (from llama_index) (4.11.2)\n",
            "Requirement already satisfied: nest-asyncio in /usr/local/lib/python3.10/dist-packages (from llama_index) (1.5.7)\n",
            "Requirement already satisfied: diskcache>=5.6.1 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python) (5.6.1)\n",
            "Requirement already satisfied: transformers<5.0.0,>=4.6.0 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (4.31.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (4.66.1)\n",
            "Requirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (2.0.1+cu118)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (0.15.2+cu118)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.2.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.10.1)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (3.8.1)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (0.1.99)\n",
            "Requirement already satisfied: huggingface-hub>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (0.16.4)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence_transformers) (3.12.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence_transformers) (2.31.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence_transformers) (6.0.1)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence_transformers) (23.1)\n",
            "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /usr/local/lib/python3.10/dist-packages (from langchain>=0.0.262->llama_index) (3.8.5)\n",
            "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from langchain>=0.0.262->llama_index) (4.0.3)\n",
            "Requirement already satisfied: langsmith<0.1.0,>=0.0.11 in /usr/local/lib/python3.10/dist-packages (from langchain>=0.0.262->llama_index) (0.0.22)\n",
            "Requirement already satisfied: numexpr<3.0.0,>=2.8.4 in /usr/local/lib/python3.10/dist-packages (from langchain>=0.0.262->llama_index) (2.8.5)\n",
            "Requirement already satisfied: openapi-schema-pydantic<2.0,>=1.2 in /usr/local/lib/python3.10/dist-packages (from langchain>=0.0.262->llama_index) (1.2.4)\n",
            "Requirement already satisfied: pydantic<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain>=0.0.262->llama_index) (1.10.12)\n",
            "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json->llama_index) (3.20.1)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from sqlalchemy>=2.0.15->llama_index) (2.0.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence_transformers) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence_transformers) (3.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence_transformers) (3.1.2)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence_transformers) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.6.0->sentence_transformers) (3.27.2)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.6.0->sentence_transformers) (16.0.6)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (2023.6.3)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (0.13.3)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (0.3.2)\n",
            "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from typing-inspect>=0.8.0->llama_index) (1.0.0)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4->llama_index) (2.4.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->sentence_transformers) (8.1.6)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->sentence_transformers) (1.3.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama_index) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama_index) (2023.3)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence_transformers) (3.2.0)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision->sentence_transformers) (9.4.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain>=0.0.262->llama_index) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain>=0.0.262->llama_index) (3.2.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain>=0.0.262->llama_index) (6.0.4)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain>=0.0.262->llama_index) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain>=0.0.262->llama_index) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain>=0.0.262->llama_index) (1.3.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->llama_index) (1.16.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (3.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (2023.7.22)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.6.0->sentence_transformers) (2.1.3)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.6.0->sentence_transformers) (1.3.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "from dotenv import load_dotenv\n",
        "load_dotenv(override=True)\n",
        "\n",
        "OPENAI_API_KEY = os.environ[\"OPENAI_API_KEY\"]\n",
        "MONGO_URI = os.environ[\"MONGO_URI\"]\n"
      ],
      "metadata": {
        "id": "n__y-ISW0T0g"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# OPTIONAL - try this if llamaindex throws an OpenAI Authentication error\n",
        "#import openai\n",
        "#openai.api_key = OPENAI_API_KEY\n"
      ],
      "metadata": {
        "id": "826DdS7X30-d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3Bwokc2oNUdi"
      },
      "outputs": [],
      "source": [
        "\n",
        "import pymongo\n",
        "from llama_index.vector_stores.mongodb import MongoDBAtlasVectorSearch\n",
        "from llama_index.indices.vector_store.base import VectorStoreIndex\n",
        "from llama_index.storage.storage_context import StorageContext\n",
        "from llama_index.readers.file.base import SimpleDirectoryReader\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# Load the PDF into LlamaIndex docs format\n",
        "uber_docs = SimpleDirectoryReader(input_files=[\"./uber_2021.pdf\"]).load_data()\n"
      ],
      "metadata": {
        "id": "KeMUF05Ey1Am"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(len(uber_docs))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GZZmv9vpy5Qe",
        "outputId": "2b067937-0b75-432a-afbf-09af78c6c5a3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "307\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Setup the configurations for MongoDBAtlasVectorSearch\n",
        "# This class is defined at https://github.com/jerryjliu/llama_index/blob/main/llama_index/vector_stores/mongodb.py\n",
        "\n",
        "MONGODB_DB_NAME = \"LlamaIndex_sample\" # default is \"default_db\"\n",
        "MONGODB_COLLECTION_NAME = \"uber_sample\" # default is \"default_collection\"\n",
        "ATLAS_VECTOR_SEARCH_INDEX_NAME = \"LlamaIndex_vector_index\" # default is \"default\""
      ],
      "metadata": {
        "id": "XkaisfmcUPRy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mongodb_client = pymongo.MongoClient(MONGO_URI)\n",
        "\n",
        "store = MongoDBAtlasVectorSearch(mongodb_client, db_name=MONGODB_DB_NAME, collection_name=MONGODB_COLLECTION_NAME, index_name=ATLAS_VECTOR_SEARCH_INDEX_NAME)\n",
        "storage_context = StorageContext.from_defaults(vector_store=store)\n",
        "index = VectorStoreIndex.from_documents(uber_docs, storage_context=storage_context)\n"
      ],
      "metadata": {
        "id": "PXhn1iJbN8gR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "The above steps will result in the LlamaIndex docs stored in MongoDB\n",
        "\n",
        "# ![picture](https://drive.google.com/uc?id=19k9gv-uoGfV7NngYPi4gk2a64l-GNCRu)"
      ],
      "metadata": {
        "id": "LZr_y7mQVntM"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Create an Index in Atlas Vector Search\n",
        "\n",
        "\n",
        "Now go to the Search tab on Atlas to create a vector search index on your cluster.  Please refer to the [documentation](https://www.mongodb.com/docs/atlas/atlas-search/define-field-mappings-for-vector-search), [sample blog](https://www.mongodb.com/developer/products/atlas/building-generative-ai-applications-vector-search-open-source-models/#semantic-search-for-movie-recommendations) to get more details on how to define an Atlas Vector Search index.\n",
        "\n",
        "Name the index `LlamaIndex_vector_index` (as defined in ATLAS_VECTOR_SEARCH_INDEX_NAME), and create the index on the namespace `LlamaIndex_sample.uber_sample`. Finally, write the following definition in the JSON editor on MongoDB Atlas:\n",
        "\n",
        "```\n",
        "{\n",
        "  \"mappings\": {\n",
        "    \"dynamic\": true,\n",
        "    \"fields\": {\n",
        "      \"embedding\": {\n",
        "        \"dimensions\": 1536,\n",
        "        \"similarity\": \"cosine\",\n",
        "        \"type\": \"knnVector\"\n",
        "      }\n",
        "    }\n",
        "  }\n",
        "}\n",
        "\n",
        "```\n",
        "In the example, `embedding` is the name of the field that contains the embedding vector.\n",
        "\n",
        "\n",
        "\n",
        "![picture](https://drive.google.com/uc?id=1lDTzSBB_TOMiQT2zbydX7R8zVERImWNn)\n",
        "\n",
        "![picture](https://drive.google.com/uc?id=1pF9wmBvmjLMUEfA2lbYsAD9m3HY5Bho1)"
      ],
      "metadata": {
        "id": "K-u2A0y7XuAW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "response = index.as_query_engine().query(\"What was Uber's revenue?\")\n",
        "print(response)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pedifM5wxh1G",
        "outputId": "2fdb810e-4be1-4c98-dd83-9352f17aecbc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Uber's revenue for the years ended December 31, 2019, 2020, and 2021 was $13,000 million, $11,139 million, and $17,455 million, respectively.\n"
          ]
        }
      ]
    }
  ]
}