llamaIndex+mongoDB_MetadataFiltering.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyOeNlAI08MGdlxxrcCJ+Krb",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "4f07009eb4494090a27c7ba401d23de6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1634a31e03624399bfa38a7144955c1e",
              "IPY_MODEL_c7bed0d4f1474732a951d78356ec1b8c",
              "IPY_MODEL_9ea9ae34c0b54d678ebf392c1172b61a"
            ],
            "layout": "IPY_MODEL_fd89f5db05564eba84ff7d34cd6d3073"
          }
        },
        "1634a31e03624399bfa38a7144955c1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_792e3dda098248a79928e5f88b25bc34",
            "placeholder": "​",
            "style": "IPY_MODEL_3753157272164f8f8417cf7ba342c95d",
            "value": "Parsing nodes: 100%"
          }
        },
        "c7bed0d4f1474732a951d78356ec1b8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c5a6a1ba63b64fdf860271495e514021",
            "max": 307,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_de55e3a89c5b48568eb6f4bb23291068",
            "value": 307
          }
        },
        "9ea9ae34c0b54d678ebf392c1172b61a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6f50396503734e99a44c0e1227ed98fe",
            "placeholder": "​",
            "style": "IPY_MODEL_58524271ed6d4e7d90c489619ffcc92e",
            "value": " 307/307 [00:00&lt;00:00, 535.15it/s]"
          }
        },
        "fd89f5db05564eba84ff7d34cd6d3073": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "792e3dda098248a79928e5f88b25bc34": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3753157272164f8f8417cf7ba342c95d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c5a6a1ba63b64fdf860271495e514021": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "de55e3a89c5b48568eb6f4bb23291068": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6f50396503734e99a44c0e1227ed98fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "58524271ed6d4e7d90c489619ffcc92e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "baf0076d9fdc4b7e991c6b0876091f20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_66955b7dd1794c0ea577659475d730ae",
              "IPY_MODEL_cdb3607bd7324515bb51307414882897",
              "IPY_MODEL_a1dd9fa9d2b5430b85b29a83721cf15b"
            ],
            "layout": "IPY_MODEL_792b0d51873744cf9cfe99ce21c29caa"
          }
        },
        "66955b7dd1794c0ea577659475d730ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1f64929e46d24d87a44491581534c0e6",
            "placeholder": "​",
            "style": "IPY_MODEL_de1f6eccb57b4b4d9bfcd8c915f1b5ad",
            "value": "Generating embeddings: 100%"
          }
        },
        "cdb3607bd7324515bb51307414882897": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_65895ae8b29745d9918e52b3203f7286",
            "max": 410,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d115b41680754aedaa91096c17048b19",
            "value": 410
          }
        },
        "a1dd9fa9d2b5430b85b29a83721cf15b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ef5af93df5fe4e85a925921e6adcfe7a",
            "placeholder": "​",
            "style": "IPY_MODEL_8f2d9432fd994ae6b97ecf46440d4981",
            "value": " 410/410 [00:15&lt;00:00, 30.25it/s]"
          }
        },
        "792b0d51873744cf9cfe99ce21c29caa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1f64929e46d24d87a44491581534c0e6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "de1f6eccb57b4b4d9bfcd8c915f1b5ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "65895ae8b29745d9918e52b3203f7286": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d115b41680754aedaa91096c17048b19": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ef5af93df5fe4e85a925921e6adcfe7a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8f2d9432fd994ae6b97ecf46440d4981": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/prakul/MongoDB-AI-Resources/blob/main/llamaIndex%2BmongoDB_MetadataFiltering.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pymongo\n",
        "!pip install llama_index==0.9.17 pypdf\n",
        "!pip3 install python-dotenv\n",
        "!pip install openai"
      ],
      "metadata": {
        "collapsed": true,
        "id": "B4Pqel7ONrMt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "from dotenv import load_dotenv\n",
        "load_dotenv(override=True)\n",
        "\n",
        "OPENAI_API_KEY = os.environ[\"OPENAI_API_KEY\"]\n",
        "MONGO_URI = os.environ[\"MONGO_URI\"]\n"
      ],
      "metadata": {
        "id": "n__y-ISW0T0g"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# OPTIONAL - try this if llamaindex throws an OpenAI Authentication error\n",
        "#import openai\n",
        "#openai.api_key = OPENAI_API_KEY\n"
      ],
      "metadata": {
        "id": "826DdS7X30-d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "3Bwokc2oNUdi"
      },
      "outputs": [],
      "source": [
        "import pymongo\n",
        "from llama_index.vector_stores.mongodb import MongoDBAtlasVectorSearch\n",
        "from llama_index.indices.vector_store.base import VectorStoreIndex\n",
        "from llama_index.storage.storage_context import StorageContext\n",
        "from llama_index.readers.file.base import SimpleDirectoryReader\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!mkdir -p 'data/10k/'\n",
        "!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/examples/data/10k/uber_2021.pdf' -O 'data/10k/uber_2021.pdf'\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iHT-YyPLHr9x",
        "outputId": "55407635-7e92-4806-caec-ad8afb262d08"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-02-16 08:53:41--  https://raw.githubusercontent.com/run-llama/llama_index/main/docs/examples/data/10k/uber_2021.pdf\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1880483 (1.8M) [application/octet-stream]\n",
            "Saving to: ‘data/10k/uber_2021.pdf’\n",
            "\n",
            "\rdata/10k/uber_2021.   0%[                    ]       0  --.-KB/s               \rdata/10k/uber_2021. 100%[===================>]   1.79M  --.-KB/s    in 0.04s   \n",
            "\n",
            "2024-02-16 08:53:41 (48.8 MB/s) - ‘data/10k/uber_2021.pdf’ saved [1880483/1880483]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# Load the PDF into LlamaIndex docs format\n",
        "uber_docs = SimpleDirectoryReader(input_files=[\"./data/10k/uber_2021.pdf\"]).load_data()\n"
      ],
      "metadata": {
        "id": "KeMUF05Ey1Am"
      },
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print((uber_docs[0].__dict__))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GZZmv9vpy5Qe",
        "outputId": "f72c9278-4178-4fb9-f5d6-8bee6b639c6a"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'id_': '32a146f4-1bf7-42d7-9a54-ed583993a996', 'embedding': None, 'metadata': {'page_label': '1', 'file_name': 'uber_2021.pdf', 'file_path': 'uber_2021.pdf', 'file_type': 'application/pdf', 'file_size': 4557617, 'creation_date': '2024-02-16', 'last_modified_date': '2024-02-16', 'last_accessed_date': '2024-02-16'}, 'excluded_embed_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'excluded_llm_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'relationships': {}, 'hash': '13c1b36110971e4dfd25578b6fe1576129cd93785e79613c1f602a96c0c68f08', 'text': 'Uber \\n2021 \\nAnnual Report', 'start_char_idx': None, 'end_char_idx': None, 'text_template': '{metadata_str}\\n\\n{content}', 'metadata_template': '{key}: {value}', 'metadata_seperator': '\\n'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Setup the configurations for MongoDBAtlasVectorSearch\n",
        "# This class is defined at https://github.com/jerryjliu/llama_index/blob/main/llama_index/vector_stores/mongodb.py\n",
        "\n",
        "MONGODB_DB_NAME = \"LlamaIndex_sample\" # default is \"default_db\"\n",
        "MONGODB_COLLECTION_NAME = \"uber_sample_1\" # default is \"default_collection\"\n",
        "ATLAS_VECTOR_SEARCH_INDEX_NAME = \"LlamaIndex_vector_index_1\" # default is \"default\""
      ],
      "metadata": {
        "id": "XkaisfmcUPRy"
      },
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mongodb_client = pymongo.MongoClient(MONGO_URI)\n",
        "#mongodb_client.list_database_names()"
      ],
      "metadata": {
        "id": "3f_xrsNhjYoN"
      },
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.vector_stores.mongodb import MongoDBAtlasVectorSearch\n",
        "\n",
        "store = MongoDBAtlasVectorSearch(mongodb_client, db_name=MONGODB_DB_NAME, collection_name=MONGODB_COLLECTION_NAME, index_name=ATLAS_VECTOR_SEARCH_INDEX_NAME)\n",
        "storage_context = StorageContext.from_defaults(vector_store=store)\n"
      ],
      "metadata": {
        "id": "PXhn1iJbN8gR"
      },
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "index = VectorStoreIndex.from_documents(uber_docs, storage_context=storage_context, show_progress=True)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "4f07009eb4494090a27c7ba401d23de6",
            "1634a31e03624399bfa38a7144955c1e",
            "c7bed0d4f1474732a951d78356ec1b8c",
            "9ea9ae34c0b54d678ebf392c1172b61a",
            "fd89f5db05564eba84ff7d34cd6d3073",
            "792e3dda098248a79928e5f88b25bc34",
            "3753157272164f8f8417cf7ba342c95d",
            "c5a6a1ba63b64fdf860271495e514021",
            "de55e3a89c5b48568eb6f4bb23291068",
            "6f50396503734e99a44c0e1227ed98fe",
            "58524271ed6d4e7d90c489619ffcc92e",
            "baf0076d9fdc4b7e991c6b0876091f20",
            "66955b7dd1794c0ea577659475d730ae",
            "cdb3607bd7324515bb51307414882897",
            "a1dd9fa9d2b5430b85b29a83721cf15b",
            "792b0d51873744cf9cfe99ce21c29caa",
            "1f64929e46d24d87a44491581534c0e6",
            "de1f6eccb57b4b4d9bfcd8c915f1b5ad",
            "65895ae8b29745d9918e52b3203f7286",
            "d115b41680754aedaa91096c17048b19",
            "ef5af93df5fe4e85a925921e6adcfe7a",
            "8f2d9432fd994ae6b97ecf46440d4981"
          ]
        },
        "id": "9l4FkTT1Cw-3",
        "outputId": "d74b8779-9a1f-4571-8317-aa712336b147"
      },
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Parsing nodes:   0%|          | 0/307 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4f07009eb4494090a27c7ba401d23de6"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating embeddings:   0%|          | 0/410 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "baf0076d9fdc4b7e991c6b0876091f20"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The above steps will result in the LlamaIndex docs stored in MongoDB\n",
        "\n",
        "# ![picture](https://drive.google.com/uc?id=19k9gv-uoGfV7NngYPi4gk2a64l-GNCRu)"
      ],
      "metadata": {
        "id": "LZr_y7mQVntM"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Create an Index in Atlas Vector Search\n",
        "\n",
        "\n",
        "Now go to the Search tab on Atlas to create a vector search index on your cluster.  Please refer to the [documentation](https://www.mongodb.com/docs/atlas/atlas-search/define-field-mappings-for-vector-search), [sample blog](https://www.mongodb.com/developer/products/atlas/building-generative-ai-applications-vector-search-open-source-models/#semantic-search-for-movie-recommendations) to get more details on how to define an Atlas Vector Search index.\n",
        "\n",
        "Name the index `LlamaIndex_vector_index` (as defined in ATLAS_VECTOR_SEARCH_INDEX_NAME), and create the index on the namespace `LlamaIndex_sample.uber_sample`. Finally, write the following definition in the JSON editor on MongoDB Atlas:\n",
        "\n",
        "```\n",
        "{\n",
        "  \"fields\": [\n",
        "    {\n",
        "      \"type\": \"vector\",\n",
        "      \"path\": \"embedding\",\n",
        "      \"numDimensions\": 1536,\n",
        "      \"similarity\": \"cosine\"\n",
        "    },\n",
        "    {\n",
        "      \"type\": \"filter\",\n",
        "      \"path\": \"metadata.page_label\"\n",
        "    }\n",
        "  ]\n",
        "}\n",
        "\n",
        "```\n",
        "In the example, `embedding` is the name of the field that contains the embedding vector.\n"
      ],
      "metadata": {
        "id": "K-u2A0y7XuAW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "response = index.as_query_engine().query(\"What was Uber's revenue?\", )\n",
        "print(response)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pedifM5wxh1G",
        "outputId": "8609ea69-c101-4958-fe9e-baa2c272279b"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Uber's revenue was $17,455 million.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(response.source_nodes)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R7to3GBNKOZq",
        "outputId": "3d1197d6-80da-4f9f-c794-e51b04a99f42"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[NodeWithScore(node=TextNode(id_='fc2b9b5b-ae48-4b9c-a5e1-85ce0af33b52', embedding=None, metadata={'page_label': '129', 'file_name': 'uber_2021.pdf', 'file_path': 'data/10k/uber_2021.pdf', 'file_type': 'application/pdf', 'file_size': 1880483, 'creation_date': '2024-02-16', 'last_modified_date': '2024-02-16', 'last_accessed_date': '2024-02-16'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='4df95547-bb0b-4602-b545-60e63f0ba785', node_type=<ObjectType.DOCUMENT: '4'>, metadata={'page_label': '129', 'file_name': 'uber_2021.pdf', 'file_path': 'data/10k/uber_2021.pdf', 'file_type': 'application/pdf', 'file_size': 1880483, 'creation_date': '2024-02-16', 'last_modified_date': '2024-02-16', 'last_accessed_date': '2024-02-16'}, hash='9d407529fd104f635e31ceea3a382998f8ad87d30601182cfd278fa840406092'), <NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(node_id='e6f840a4-97ef-4c7b-8de9-eb5645dc2254', node_type=<ObjectType.TEXT: '1'>, metadata={'page_label': '128', 'file_name': 'uber_2021.pdf', 'file_path': 'data/10k/uber_2021.pdf', 'file_type': 'application/pdf', 'file_size': 1880483, 'creation_date': '2024-02-16', 'last_modified_date': '2024-02-16', 'last_accessed_date': '2024-02-16'}, hash='34a23be1517645954a0a7cbf689454920d5ef5b1fe89fd5c501951de7e20af96'), <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='7f7ba98f-09fd-4792-a1b2-d2413b201eff', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='78ac114d168fc0dbb1a2c480568f582dfad34cd1bf863ed13b6b132c0c883772')}, hash='5651e31ec38bafa10e57427b5389b7bda1f190c14d3d029a14504529f5e8c541', text=\"Year Ended December 31,2019\\n2020 2021 United States\\n$ 7,968 $ 6,082 $ 9,058 All other countries\\n5,032 5,057 8,397 Total Revenue\\n$ 13,000 $ 11,139 $ 17,455 As of December 31,\\n2020\\n2021 United States\\n$ 2,940 $ 2,991 All other countries\\n252 250 Total long-lived assets, net\\n$ 3,192 $ 3,241 Revenue grouped by offerings is inc\\nluded in Note 2 – Revenue.Note 15 – Commitments and Contingencies\\nContingencies\\nFrom time to time, we are a party to various claims, non-income tax audits and litigation in the normal course of b\\nusiness. As of December 31, 2020 and 2021,we\\n had  recorded  aggregate  liabilities  of  $1.8  billion  and  $2.2  billion,  respectively,  of  which  $1.3  billion  and  $1.3  billion  relate  to  non-income  tax  matters,respectively,\\n in  accrued  and  other  current  liabilities  on  the  consolidated  balance  sheets  for  all  of  our  legal,  regulatory  and  non-income  tax  matters  that  wereprobable and reasonably est\\nimable.We\\n are  currently  party  to  various  legal  and  regulatory  matters  that  have  arisen  in  the  normal  course  of  business  and  include,  among  others,  allegedindependent\\n contractor misclassification claims, Fair Credit Reporting Act (“FCRA”) claims, alleged background check violations, pricing and advertising claims,unfair\\n competition claims, intellectual property claims, employment discrimination and other employment-related claims, Telephone Consumer Protection Act(“TCPA”)\\n claims, Americans with Disabilities Act (“ADA”) claims, data and privacy claims, securities claims, antitrust claims, challenges to regulations, andother\\n matters. We have existing litigation, including class actions, Private Attorney General Act lawsuits, arbitration claims, and governmental administrative andaudit\\n proceedings,  asserting  claims  by  or  on  behalf  of  Drivers  that  Drivers  are  misclassified  as  independent  contractors.  In  connection  with  the  enactment  ofCalifornia\\n State Assembly Bill 5 (“AB5”), we have received and expect to continue to receive - in California and in other jurisdictions - an increased number ofmisclassification\\n claims. With respect to our outstanding legal and regulatory matters, based on our current knowledge, we believe that the ultimate amount orrange\\n of reasonably possible loss will not, either individually or in the aggregate, have a material adverse effect on our business, financial position, results ofoperations,\\n or cash flows. The outcome of such legal matters is inherently unpredictable and subject to significant uncertainties. If one or more of these matterswere\\n resolved  against  us  for  amounts  in  excess  of  management's  expectations,  our  results  of  operations,  financial  condition  or  cash  flows  could  be  materiallyadversely affected.\\nDriver Classification\\nCalifornia Attorney General Lawsuit\\nIn\\n January 2020, AB5 went into effect. AB5 codifies a test to determine whether a worker is an employee under California law. The test is referred to as the“ABC”\\n test, and was originally handed down by the California Supreme Court in Dynamex Operations v. Superior Court in 2018. Under the ABC test, workersperforming\\n services  for  a  hiring  entity  are  considered  employees  unless  the  hiring  entity  can  demonstrate  three  things:  the  worker  (A)  is  free  from  the  hiringentity’s\\n control, (B) performs work that is outside the usual course of the hiring entity’s business, and (C) customarily engages in the independent trade, work ortype of business performed fo\\nr the hiring entity.On\\n May 5, 2020, the California Attorney General, in conjunction with the city attorneys for San Francisco, Los Angeles and San Diego, filed a complaint inSan\\n Francisco Superior Court against Uber and Lyft, Inc. (“Lyft”). The complaint alleges drivers are misclassified, and seeks an injunction and monetary damagesrelated to the alleged competitive\\n advantage caused by the alleged misclassification of drivers.On\\n August 10, 2020, the Court issued a preliminary injunction order, prohibiting us from classifying drivers as independent contractors and from violatingvarious wage and hour laws. The injunction was stayed pending appeal.\", start_char_idx=0, end_char_idx=4191, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=0.9358124732971191), NodeWithScore(node=TextNode(id_='cde39828-3f04-42f1-8857-1239aa58f65b', embedding=None, metadata={'page_label': '77', 'file_name': 'uber_2021.pdf', 'file_path': 'data/10k/uber_2021.pdf', 'file_type': 'application/pdf', 'file_size': 1880483, 'creation_date': '2024-02-16', 'last_modified_date': '2024-02-16', 'last_accessed_date': '2024-02-16'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='d045d18f-b064-4c7f-8f73-355b6d510c6c', node_type=<ObjectType.DOCUMENT: '4'>, metadata={'page_label': '77', 'file_name': 'uber_2021.pdf', 'file_path': 'data/10k/uber_2021.pdf', 'file_type': 'application/pdf', 'file_size': 1880483, 'creation_date': '2024-02-16', 'last_modified_date': '2024-02-16', 'last_accessed_date': '2024-02-16'}, hash='d3c195bed447b4c1c432535b761ab8d3c029b2c78567944661a49965f933ec67'), <NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(node_id='b0af9047-a41b-4620-bfef-5086b7f567a7', node_type=<ObjectType.TEXT: '1'>, metadata={'page_label': '76', 'file_name': 'uber_2021.pdf', 'file_path': 'data/10k/uber_2021.pdf', 'file_type': 'application/pdf', 'file_size': 1880483, 'creation_date': '2024-02-16', 'last_modified_date': '2024-02-16', 'last_accessed_date': '2024-02-16'}, hash='918763fbe314b84c16a09c769fe18d71b8ebfb2398ab4f74508ac694389ed337'), <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='6cb4aa09-eecc-4c22-9157-37d7cb2d0dc1', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='8288651338b6653592207a636b3e34bf52524d595ef86ad37903d9b40e62cab4')}, hash='4c4ed7a9f30d0bfecf203f20c82d1523e74bec51bed7faa5afc73b915c3ad72b', text='UBER TECHNOLOGIES, INC.CONSOLIDATED STATEMENTS OF\\n OPERATIONS(In millions, except share amounts which are ref\\nlected in thousands, and per share amounts)Year Ended December 31,\\n2019\\n2020 2021 Revenue\\n$ 13,000 $ 11,139 $ 17,455 Costs and expenses\\nCost of revenue, exclusive of dep\\nreciation and amortization shown separately below6,061 5,154 9,351 Operations and support\\n2,302 1,819 1,877 Sales and marketing\\n4,626 3,583 4,789 Research and development\\n4,836 2,205 2,054 General and administrative\\n3,299 2,666 2,316 Depreciation and amortization\\n472 575 902 Total costs and expenses\\n21,596 16,002 21,289 Loss from operations\\n(8,596) (4,863) (3,834) Interest expense\\n(559) (458) (483) Other income (expense), net\\n722 (1,625) 3,292 Loss before income taxes and loss from equity me\\nthod investments(8,433) (6,946) (1,025) Provision for (benefit fro\\nm) income taxes45 (192) (492) Loss from equity method invest\\nments(34) (34) (37) Net loss including non-controlling interests\\n(8,512) (6,788) (570) Less: net loss attributab\\nle to non-controlling interests, net of tax(6) (20) (74) Net loss attributable to Uber Technologies, Inc.\\n$ (8,506) $ (6,768) $ (496) Net loss per share attributable to Uber Technologies, Inc. co\\nmmon stockholders:Basic\\n$ (6.81) $ (3.86) $ (0.26) Diluted\\n$ (6.81) $ (3.86) $ (0.29) Weighted-average shares used to compute net loss per shar\\ne attributable to common stockholders:Basic\\n1,248,353 1,752,960 1,892,546 Diluted\\n1,248,353 1,752,960 1,895,519 The accompanying notes are an integral part of \\nthese consolidated financial statements.75', start_char_idx=0, end_char_idx=1560, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=0.9347857236862183)]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Metadata Filtering\n",
        "\n",
        "Each LlamaIndex document contains a metadata field.\n",
        "The default metadata field contains following fields\n",
        "\n",
        "```\n",
        "metadata={'page_label': '129', 'file_name': 'uber_2021.pdf', 'file_path': 'data/10k/uber_2021.pdf', 'file_type': 'application/pdf', 'file_size': 1880483, 'creation_date': '2024-02-16', 'last_modified_date': '2024-02-16', 'last_accessed_date': '2024-02-16'}\n",
        "```\n",
        "\n",
        "\n",
        "The metadata can be customized using instructions at\n",
        "https://docs.llamaindex.ai/en/stable/module_guides/loading/documents_and_nodes/usage_documents.html\n",
        "\n",
        "\n",
        "Whatever metadata field is desired to be searchable, needs to be included in the Atlas Vector Search index definition. In the example above we have included the field `metadata.page_label` in the Vector Search index definition via the `type: filter`\n",
        "```\n",
        "{\n",
        "      \"type\": \"filter\",\n",
        "      \"path\": \"metadata.page_label\"\n",
        "    }\n",
        "```\n",
        "\n",
        "In the above response, we can see that the answer was contained in the page_label = 129.\n",
        "\n",
        "If we exclude that particular page_label we wont be able to find the corresponding answer"
      ],
      "metadata": {
        "id": "u-gENDA_LjKf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.vector_stores import ExactMatchFilter, MetadataFilters\n",
        "\n",
        "filters = MetadataFilters(\n",
        "    filters=[ExactMatchFilter(key=\"metadata.page_label\", value=\"131\")]\n",
        ")\n",
        "response = index.as_query_engine(filters=filters).query(\"What was Uber's revenue?\", )\n",
        "print(response)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zTh66z7aGQgs",
        "outputId": "1b6eb9db-afc8-4368-aaf4-581eddf8f133"
      },
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "I'm sorry, but the given context does not provide any information about Uber's revenue.\n"
          ]
        }
      ]
    }
  ]
}