[new_version]_llamaIndex+mongoDB_MetadataFiltering.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyPQYK4riLMaR0nXF3rmkHEY",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "c56e1204fc7c4f0d8de75e79a56dfefc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4cd486b581b44ecab16bacd84e42591d",
              "IPY_MODEL_1041850445234fa581e3074e88f94524",
              "IPY_MODEL_41ef2fff71194ce18a1a4f71b06c82e2"
            ],
            "layout": "IPY_MODEL_cecc7ac3b4584e4695c02138a1222495"
          }
        },
        "4cd486b581b44ecab16bacd84e42591d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5793479ff3b846dc844bd1f41bf96344",
            "placeholder": "​",
            "style": "IPY_MODEL_4ecdb7fb7f5b4f3b823715ad21bf32af",
            "value": "Parsing nodes: 100%"
          }
        },
        "1041850445234fa581e3074e88f94524": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_89a402f9de314b6e80bacfe6826d86c6",
            "max": 307,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_130345e1795c4877b7b25f1484179fcd",
            "value": 307
          }
        },
        "41ef2fff71194ce18a1a4f71b06c82e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e05288b914be4e3883479d2116392281",
            "placeholder": "​",
            "style": "IPY_MODEL_ff5bde828b544ec8849785d2dbbeae67",
            "value": " 307/307 [00:01&lt;00:00, 179.74it/s]"
          }
        },
        "cecc7ac3b4584e4695c02138a1222495": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5793479ff3b846dc844bd1f41bf96344": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4ecdb7fb7f5b4f3b823715ad21bf32af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "89a402f9de314b6e80bacfe6826d86c6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "130345e1795c4877b7b25f1484179fcd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e05288b914be4e3883479d2116392281": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ff5bde828b544ec8849785d2dbbeae67": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "29b19af2908448ec8f16e36b4ca47254": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_27c0460d0dfa425fb42e7aff755a25a2",
              "IPY_MODEL_be1deb9de3ef4dda8eaddd8a703aeec4",
              "IPY_MODEL_0f546d31272e40c080ead6edd77392cc"
            ],
            "layout": "IPY_MODEL_b1d330945ed740f3ad543e9ed24e3307"
          }
        },
        "27c0460d0dfa425fb42e7aff755a25a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_75dba2bdf65142c1be0934b38b7e7b5b",
            "placeholder": "​",
            "style": "IPY_MODEL_5eb77804a69e4bfc902b69952cd65ebf",
            "value": "Generating embeddings: 100%"
          }
        },
        "be1deb9de3ef4dda8eaddd8a703aeec4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_566c6939daed474b9bca19a33e4b9e26",
            "max": 410,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8aaacc22b5b24c69b8921311845a1e70",
            "value": 410
          }
        },
        "0f546d31272e40c080ead6edd77392cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ebbe69adaf704c63824e8d911885b6bb",
            "placeholder": "​",
            "style": "IPY_MODEL_9c19c8b62fbb4ecd9c268ffa5c718364",
            "value": " 410/410 [00:09&lt;00:00, 40.09it/s]"
          }
        },
        "b1d330945ed740f3ad543e9ed24e3307": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "75dba2bdf65142c1be0934b38b7e7b5b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5eb77804a69e4bfc902b69952cd65ebf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "566c6939daed474b9bca19a33e4b9e26": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8aaacc22b5b24c69b8921311845a1e70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ebbe69adaf704c63824e8d911885b6bb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9c19c8b62fbb4ecd9c268ffa5c718364": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/prakul/MongoDB-AI-Resources/blob/main/%5Bnew_version%5D_llamaIndex%2BmongoDB_MetadataFiltering.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip3 install python-dotenv\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "60cGV4I5beyF",
        "outputId": "8ba80d9d-2472-497e-f427-f78b1393c651"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting python-dotenv\n",
            "  Downloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n",
            "Installing collected packages: python-dotenv\n",
            "Successfully installed python-dotenv-1.0.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install llama-index\n",
        "!pip install llama-index-vector-stores-mongodb\n",
        "!pip install llama-index-embeddings-openai\n",
        "!pip install pymongo\n",
        "!pip install datasets\n",
        "!pip install pandas\n",
        "!pip3 install python-dotenv\n"
      ],
      "metadata": {
        "collapsed": true,
        "id": "B4Pqel7ONrMt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "from dotenv import load_dotenv\n",
        "load_dotenv(override=True)\n",
        "\n",
        "OPENAI_API_KEY = os.environ[\"OPENAI_API_KEY\"]\n",
        "MONGO_URI = os.environ[\"MONGO_URI\"]\n"
      ],
      "metadata": {
        "id": "n__y-ISW0T0g"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.core.settings import Settings\n",
        "from llama_index.llms.openai import OpenAI\n",
        "from llama_index.embeddings.openai import OpenAIEmbedding\n",
        "\n",
        "embed_model = OpenAIEmbedding(model=\"text-embedding-3-small\", dimensions=256)\n",
        "llm = OpenAI()\n",
        "\n",
        "Settings.llm = llm\n",
        "Settings.embed_model = embed_model"
      ],
      "metadata": {
        "id": "Jkb26yvuUU52"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# OPTIONAL - try this if llamaindex throws an OpenAI Authentication error\n",
        "#import openai\n",
        "#openai.api_key = OPENAI_API_KEY\n"
      ],
      "metadata": {
        "id": "826DdS7X30-d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "3Bwokc2oNUdi"
      },
      "outputs": [],
      "source": [
        "import pymongo\n",
        "\n",
        "from llama_index.core import SimpleDirectoryReader\n",
        "#index = VectorStoreIndex.from_vector_store(vector_store)\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!mkdir -p 'data/10k/'\n",
        "!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/examples/data/10k/uber_2021.pdf' -O 'data/10k/uber_2021.pdf'\n"
      ],
      "metadata": {
        "id": "iHT-YyPLHr9x"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# Load the PDF into LlamaIndex docs format\n",
        "uber_docs = SimpleDirectoryReader(input_files=[\"./data/10k/uber_2021.pdf\"]).load_data()\n"
      ],
      "metadata": {
        "id": "KeMUF05Ey1Am"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print((uber_docs[0].__dict__))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GZZmv9vpy5Qe",
        "outputId": "f72c9278-4178-4fb9-f5d6-8bee6b639c6a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'id_': '32a146f4-1bf7-42d7-9a54-ed583993a996', 'embedding': None, 'metadata': {'page_label': '1', 'file_name': 'uber_2021.pdf', 'file_path': 'uber_2021.pdf', 'file_type': 'application/pdf', 'file_size': 4557617, 'creation_date': '2024-02-16', 'last_modified_date': '2024-02-16', 'last_accessed_date': '2024-02-16'}, 'excluded_embed_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'excluded_llm_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'relationships': {}, 'hash': '13c1b36110971e4dfd25578b6fe1576129cd93785e79613c1f602a96c0c68f08', 'text': 'Uber \\n2021 \\nAnnual Report', 'start_char_idx': None, 'end_char_idx': None, 'text_template': '{metadata_str}\\n\\n{content}', 'metadata_template': '{key}: {value}', 'metadata_seperator': '\\n'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Setup the configurations for MongoDBAtlasVectorSearch\n",
        "# This class is defined at https://github.com/jerryjliu/llama_index/blob/main/llama_index/vector_stores/mongodb.py\n",
        "\n",
        "MONGODB_DB_NAME = \"LlamaIndex_sample\" # default is \"default_db\"\n",
        "MONGODB_COLLECTION_NAME = \"uber_sample_3\" # default is \"default_collection\"\n",
        "ATLAS_VECTOR_SEARCH_INDEX_NAME = \"LlamaIndex_vector_index_2\" # default is \"default\""
      ],
      "metadata": {
        "id": "XkaisfmcUPRy"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mongodb_client = pymongo.MongoClient(MONGO_URI)\n",
        "#mongodb_client.list_database_names()"
      ],
      "metadata": {
        "id": "3f_xrsNhjYoN"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.vector_stores.mongodb import MongoDBAtlasVectorSearch\n",
        "from llama_index.core import VectorStoreIndex, StorageContext\n",
        "\n",
        "store = MongoDBAtlasVectorSearch(mongodb_client, db_name=MONGODB_DB_NAME, collection_name=MONGODB_COLLECTION_NAME, index_name=ATLAS_VECTOR_SEARCH_INDEX_NAME)\n",
        "storage_context = StorageContext.from_defaults(vector_store=store)\n"
      ],
      "metadata": {
        "id": "PXhn1iJbN8gR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Process the PDF and store in Atlas Vector Search"
      ],
      "metadata": {
        "id": "3kzVd0ql_C7x"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#index = VectorStoreIndex.from_documents(uber_docs, storage_context=storage_context, show_progress=True)\n",
        "\n",
        "index = VectorStoreIndex.from_documents(\n",
        "    uber_docs, storage_context=storage_context, show_progress=True\n",
        ")\n",
        "\n",
        "#index = VectorStoreIndex.from_documents(uber_docs, storage_context=storage_context, show_progress=True)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "c56e1204fc7c4f0d8de75e79a56dfefc",
            "4cd486b581b44ecab16bacd84e42591d",
            "1041850445234fa581e3074e88f94524",
            "41ef2fff71194ce18a1a4f71b06c82e2",
            "cecc7ac3b4584e4695c02138a1222495",
            "5793479ff3b846dc844bd1f41bf96344",
            "4ecdb7fb7f5b4f3b823715ad21bf32af",
            "89a402f9de314b6e80bacfe6826d86c6",
            "130345e1795c4877b7b25f1484179fcd",
            "e05288b914be4e3883479d2116392281",
            "ff5bde828b544ec8849785d2dbbeae67",
            "29b19af2908448ec8f16e36b4ca47254",
            "27c0460d0dfa425fb42e7aff755a25a2",
            "be1deb9de3ef4dda8eaddd8a703aeec4",
            "0f546d31272e40c080ead6edd77392cc",
            "b1d330945ed740f3ad543e9ed24e3307",
            "75dba2bdf65142c1be0934b38b7e7b5b",
            "5eb77804a69e4bfc902b69952cd65ebf",
            "566c6939daed474b9bca19a33e4b9e26",
            "8aaacc22b5b24c69b8921311845a1e70",
            "ebbe69adaf704c63824e8d911885b6bb",
            "9c19c8b62fbb4ecd9c268ffa5c718364"
          ]
        },
        "id": "9l4FkTT1Cw-3",
        "outputId": "c2659145-c6bf-439a-b920-7df092263e2d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Parsing nodes:   0%|          | 0/307 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c56e1204fc7c4f0d8de75e79a56dfefc"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating embeddings:   0%|          | 0/410 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "29b19af2908448ec8f16e36b4ca47254"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The above steps will result in the LlamaIndex docs stored in MongoDB\n",
        "\n",
        "# ![picture](https://drive.google.com/uc?id=19k9gv-uoGfV7NngYPi4gk2a64l-GNCRu)"
      ],
      "metadata": {
        "id": "LZr_y7mQVntM"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Create an Index in Atlas Vector Search\n",
        "\n",
        "\n",
        "Now go to the Search tab on Atlas to create a vector search index on your cluster.  Please refer to the [documentation](https://www.mongodb.com/docs/atlas/atlas-search/define-field-mappings-for-vector-search), [sample blog](https://www.mongodb.com/developer/products/atlas/building-generative-ai-applications-vector-search-open-source-models/#semantic-search-for-movie-recommendations) to get more details on how to define an Atlas Vector Search index.\n",
        "\n",
        "Name the index `LlamaIndex_vector_index` (as defined in ATLAS_VECTOR_SEARCH_INDEX_NAME), and create the index on the namespace `LlamaIndex_sample.uber_sample`. Finally, write the following definition in the JSON editor on MongoDB Atlas:\n",
        "\n",
        "```\n",
        "{\n",
        "  \"fields\": [\n",
        "    {\n",
        "      \"type\": \"vector\",\n",
        "      \"path\": \"embedding\",\n",
        "      \"numDimensions\": 256,\n",
        "      \"similarity\": \"cosine\"\n",
        "    },\n",
        "    {\n",
        "      \"type\": \"filter\",\n",
        "      \"path\": \"metadata.page_label\"\n",
        "    }\n",
        "  ]\n",
        "}\n",
        "\n",
        "```\n",
        "In the example, `embedding` is the name of the field that contains the embedding vector.\n"
      ],
      "metadata": {
        "id": "K-u2A0y7XuAW"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "##  Initialize Index (Optional)\n",
        "\n",
        "For data already in MongoDB you can initialize the index like this"
      ],
      "metadata": {
        "id": "vjiOnbifLlMG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.core import VectorStoreIndex, StorageContext\n",
        "vector_store = MongoDBAtlasVectorSearch(mongodb_client, db_name=MONGODB_DB_NAME, collection_name=MONGODB_COLLECTION_NAME, index_name=ATLAS_VECTOR_SEARCH_INDEX_NAME)\n",
        "index = VectorStoreIndex.from_vector_store(vector_store)\n"
      ],
      "metadata": {
        "id": "P4zJqB5uLtQR"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Performing Semantic Search\n",
        "\n",
        "[Reference](https://docs.llamaindex.ai/en/stable/module_guides/querying/retriever/root.html)"
      ],
      "metadata": {
        "id": "AZNSx_GYNkBz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "retriever = index.as_retriever(similarity_top_k=3)\n",
        "nodes = retriever.retrieve(\"What was Uber's revenue?\")\n"
      ],
      "metadata": {
        "id": "-TOWDr6ZNIhS"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for node in nodes:\n",
        "    print(node)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AU2R1mPGNWgN",
        "outputId": "91e6b0e5-e860-4ea0-a11c-67c31bfbd6dc"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Node ID: 681d2a66-97ef-486d-820c-25c1ec0efe8d\n",
            "Text: Financial and Operational HighlightsYear Ended December 31,\n",
            "Constant Currency (In millions, except percentages) 2020 2021 2020 to\n",
            "2021 %Change 2020 to 2021 % Change Monthly Active Platform Consumers\n",
            "(“MAPCs”) 93 118 27 %Trips  5,025 6,368 27 %Gross Bookings  $ 57,897 $\n",
            "90,415 56 %53 % Revenue $ 11,139 $ 17,455 57 %54 % Net loss\n",
            "attributable to ...\n",
            "Score:  0.855\n",
            "\n",
            "Node ID: 2b04e088-7d95-4693-95c6-9976fc387c56\n",
            "Text: UBER TECHNOLOGIES, INC.CONSOLIDATED STATEMENTS OF  OPERATIONS(In\n",
            "millions, except share amounts which are ref lected in thousands, and\n",
            "per share amounts)Year Ended December 31, 2019 2020 2021 Revenue $\n",
            "13,000 $ 11,139 $ 17,455 Costs and expenses Cost of revenue, exclusive\n",
            "of dep reciation and amortization shown separately below6,061 5,154\n",
            "9,351 ...\n",
            "Score:  0.851\n",
            "\n",
            "Node ID: 824a3448-ebfa-4b9c-9437-d18684414670\n",
            "Text: Through our partnership with Arizona State University, over4,000\n",
            "Drivers and their fami ly members have enrolled in an undergraduate\n",
            "degree program.• Engagement:  We are focused on listening to and\n",
            "responding to the ideas and concerns of Drivers and Merchants who use\n",
            "our platform. We believe thatthe  best ideas can come from anywhere,\n",
            "both insid...\n",
            "Score:  0.848\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Performing Semantic Search with Metadata filtering"
      ],
      "metadata": {
        "id": "o4qPp-EzOZmz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.core.vector_stores import (\n",
        "    MetadataFilter,\n",
        "    MetadataFilters,\n",
        "    ExactMatchFilter,\n",
        "    FilterOperator,\n",
        ")\n",
        "filters = MetadataFilters(\n",
        "    filters=[ExactMatchFilter(key=\"metadata.page_label\", value=\"131\")]\n",
        ")\n",
        "retriever = index.as_retriever(similarity_top_k=3, filters=filters)\n",
        "nodes = retriever.retrieve(\"What was Uber's revenue?\")"
      ],
      "metadata": {
        "id": "LedM9zYXOeHV"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for node in nodes:\n",
        "    print(node)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JWkL-J7PO1KD",
        "outputId": "149560ae-68e9-482f-b88a-efb2f940f729"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Node ID: e9ae9536-2a57-4a05-a840-48fe1601a18c\n",
            "Text: Other Driver Classification MattersAdditionally, we  have\n",
            "received other lawsuits and governmental inquiries in other\n",
            "jurisdictions, and anticipate future claims, lawsuits, arbitration\n",
            "proceedings,administrative  actions,  and  government  investigations\n",
            "and  audits  challenging  our  classification  of  Drivers  as\n",
            "independent  contractors  a...\n",
            "Score:  0.770\n",
            "\n",
            "Node ID: 0630edba-db9c-4b6f-8be1-751024049a5d\n",
            "Text: Uber filed a proof of claim in the bankruptcy court, and Levando\n",
            "wski additionally asserted a claim against Uber alleging that Uber\n",
            "failed toperform its obligations under an agreement with Otto Trucking\n",
            ", LLC. For these claims, Uber and Levandowski reached a confidential\n",
            "settlement in principle that isscheduled  for an approval hearing with\n",
            "the ...\n",
            "Score:  0.759\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Performing RAG"
      ],
      "metadata": {
        "id": "usM5xicEOVdb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "response = index.as_query_engine().query(\"What was Uber's revenue?\", )\n",
        "print(response)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pedifM5wxh1G",
        "outputId": "6ae1547f-1124-4464-a4ab-498d5156b2d5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Uber's revenue for the year ended December 31, 2021, was $17.5 billion.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(response.source_nodes)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R7to3GBNKOZq",
        "outputId": "0ba3f2b9-c9eb-4daf-f4d3-9dfffc66486f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[NodeWithScore(node=TextNode(id_='681d2a66-97ef-486d-820c-25c1ec0efe8d', embedding=None, metadata={'page_label': '53', 'file_name': 'uber_2021.pdf', 'file_path': 'data/10k/uber_2021.pdf', 'file_type': 'application/pdf', 'file_size': 1880483, 'creation_date': '2024-02-16', 'last_modified_date': '2024-02-16', 'last_accessed_date': '2024-02-16'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='f1ae6bfd-ad27-4da7-940f-fbedf7273e28', node_type=<ObjectType.DOCUMENT: '4'>, metadata={'page_label': '53', 'file_name': 'uber_2021.pdf', 'file_path': 'data/10k/uber_2021.pdf', 'file_type': 'application/pdf', 'file_size': 1880483, 'creation_date': '2024-02-16', 'last_modified_date': '2024-02-16', 'last_accessed_date': '2024-02-16'}, hash='27204e1302bb751bb2541925e57008cb61ff9deddf19fd9dc48997dffcb24e29'), <NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(node_id='e4b4dd17-ed46-4f3a-a826-0595b1a45f33', node_type=<ObjectType.TEXT: '1'>, metadata={'page_label': '52', 'file_name': 'uber_2021.pdf', 'file_path': 'data/10k/uber_2021.pdf', 'file_type': 'application/pdf', 'file_size': 1880483, 'creation_date': '2024-02-16', 'last_modified_date': '2024-02-16', 'last_accessed_date': '2024-02-16'}, hash='aec9a6ab6d29f02186f3c994ed9a0a2f2d6eb365854418c8fd7cdf315f0e59ab'), <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='15968f82-9424-42c5-a82e-9b9af0f2f4cf', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='aa569a98d77942a3704e8a387f35e749c08a7e5be3539d104fabaf9985d8b893')}, text='Financial and Operational HighlightsYear Ended December 31,\\nConstant Currency (In millions, except percentages)\\n2020 2021 2020 to 2021 %Change\\n2020 to 2021 % Change Monthly Active Platform Consumers \\n(“MAPCs”) 93 118 27 %Trips \\n5,025 6,368 27 %Gross Bookings \\n$ 57,897 $ 90,415 56 %53 % Revenue\\n$ 11,139 $ 17,455 57 %54 % Net loss attributable to Ube\\nr Technologies, Inc. $ (6,768) $ (496) 93 %Mobility Adjusted EBITDA\\n$ 1,169 $ 1,596 37 %Delivery Adjusted EBITDA\\n$ (873) $ (348) 60 %Adjusted EBITDA \\n$ (2,528) $ (774) 69 % See\\n the section titled “Reconciliations of Non-GAAP Financial Measures” for more information and reconciliations to the most directly comparable GAAPfinancial measure.\\n See the section titled \\n“Certain Key Metrics and Non-GAAP Financial Measures” below for more information.MAPCs presented for annual periods ar\\ne MAPCs for the fourth quarter of the year.Net loss attributable to Ube\\nr Technologies, Inc. includes stock-based compensation expense of $827 million and $1.2 billion during the years endedDecember 31, 2020 and 2021, respectiv\\nely.Highlights for 2021\\nOverall\\n Gross Bookings increased by $32.5 billion in 2021, up 56%, or 53% on a constant currency basis, compared to 2020. Delivery Gross Bookings grew66% from\\n 2020, on a constant currency basis, due to an increase in food delivery orders and higher basket sizes as a result of stay-at-home order demand related toCOVID-19, as\\n well as continued expansion across U.S. and international markets. Additionally, we saw an increase in Delivery revenue resulting from an increasein\\n certain Courier payments and incentives that are recorded in cost of revenue, where we are primarily responsible for delivery services and pay Couriers forservices provided. Mobility Gross Bookings g\\nrew 36%, on a constant currency basis, from 2020, due to increases in Trip volumes as the business recovers from theimpacts of COVID-19.\\nRevenue\\n was $17.5 billion, or up 57% year-over-year, reflecting the overall growth in our Delivery business and an increase in Freight revenue attributable tothe\\n acquisition of Transplace in the fourth quarter of 2021 as well as growth in the number of shippers and carriers on the network combined with an increase involumes with our top shippers.\\nNet\\n loss attributable to Uber Technologies, Inc. was $496 million, a 93% improvement year-over-year, driven by a $1.6 billion pre-tax gain on the sale of ourATG\\n Business to Aurora, a $1.6 billion pre-tax  net benefit relating to Uber’s equity investments, as  well as reductions in our fixed cost structure and increasedvariable cost effi\\nciencies. Net loss attributable to Uber Technologies, Inc. also included $1.2 billion of stock-based compensation expense.Adjusted\\n EBITDA loss was $774 million, improving $1.8 billion from 2020 with Mobility Adjusted EBITDA profit of $1.6 billion. Additionally, DeliveryAdjusted\\n EBITDA loss of $348 million, improved $525 million and Delivery Adjusted EBITDA margin as a percentage of Delivery Gross Bookings improved to(0.7)% from (2.9)%, compa\\nred to 2020.We ended the year with $4.3 bil\\nlion in cash and cash equivalents.Other Developments for 2021\\nAcquisitions\\nRemaining Interests in Cornershop\\nIn\\n August 2021, we completed the acquisition of the remaining 45% ownership interest in Cornershop Cayman (“Cornershop”), or 47%, on a fully-dilutedbasis, in an all-stock tr\\nansaction.Drizly\\nOn\\n October 12, 2021, we completed the acquisition of 100% ownership interest in The Drizly Group, Inc. (“Drizly”), an on-demand alcohol marketplace inNorth America, allowing us to expand \\nalcohol offerings in our Delivery business.(1)(2), (3)\\n(2)\\n(2)\\n(4)\\n(1), (2)\\n(1)\\n(2)\\n(3) \\n(4) \\n51', start_char_idx=0, end_char_idx=3675, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=0.8548254370689392), NodeWithScore(node=TextNode(id_='2b04e088-7d95-4693-95c6-9976fc387c56', embedding=None, metadata={'page_label': '77', 'file_name': 'uber_2021.pdf', 'file_path': 'data/10k/uber_2021.pdf', 'file_type': 'application/pdf', 'file_size': 1880483, 'creation_date': '2024-02-16', 'last_modified_date': '2024-02-16', 'last_accessed_date': '2024-02-16'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='7445803a-b316-4eae-be9a-d3f8950c1777', node_type=<ObjectType.DOCUMENT: '4'>, metadata={'page_label': '77', 'file_name': 'uber_2021.pdf', 'file_path': 'data/10k/uber_2021.pdf', 'file_type': 'application/pdf', 'file_size': 1880483, 'creation_date': '2024-02-16', 'last_modified_date': '2024-02-16', 'last_accessed_date': '2024-02-16'}, hash='d3c195bed447b4c1c432535b761ab8d3c029b2c78567944661a49965f933ec67'), <NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(node_id='79f9b73b-8e6d-4790-b188-dcb1e75fd6b4', node_type=<ObjectType.TEXT: '1'>, metadata={'page_label': '76', 'file_name': 'uber_2021.pdf', 'file_path': 'data/10k/uber_2021.pdf', 'file_type': 'application/pdf', 'file_size': 1880483, 'creation_date': '2024-02-16', 'last_modified_date': '2024-02-16', 'last_accessed_date': '2024-02-16'}, hash='6a92f47832e2e72028a25b16912c0f347837d6c2ee469bb09646bab41cd6a4b8'), <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='ea25b9c6-49be-4ece-b858-e6a4058ef500', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='8288651338b6653592207a636b3e34bf52524d595ef86ad37903d9b40e62cab4')}, text='UBER TECHNOLOGIES, INC.CONSOLIDATED STATEMENTS OF\\n OPERATIONS(In millions, except share amounts which are ref\\nlected in thousands, and per share amounts)Year Ended December 31,\\n2019\\n2020 2021 Revenue\\n$ 13,000 $ 11,139 $ 17,455 Costs and expenses\\nCost of revenue, exclusive of dep\\nreciation and amortization shown separately below6,061 5,154 9,351 Operations and support\\n2,302 1,819 1,877 Sales and marketing\\n4,626 3,583 4,789 Research and development\\n4,836 2,205 2,054 General and administrative\\n3,299 2,666 2,316 Depreciation and amortization\\n472 575 902 Total costs and expenses\\n21,596 16,002 21,289 Loss from operations\\n(8,596) (4,863) (3,834) Interest expense\\n(559) (458) (483) Other income (expense), net\\n722 (1,625) 3,292 Loss before income taxes and loss from equity me\\nthod investments(8,433) (6,946) (1,025) Provision for (benefit fro\\nm) income taxes45 (192) (492) Loss from equity method invest\\nments(34) (34) (37) Net loss including non-controlling interests\\n(8,512) (6,788) (570) Less: net loss attributab\\nle to non-controlling interests, net of tax(6) (20) (74) Net loss attributable to Uber Technologies, Inc.\\n$ (8,506) $ (6,768) $ (496) Net loss per share attributable to Uber Technologies, Inc. co\\nmmon stockholders:Basic\\n$ (6.81) $ (3.86) $ (0.26) Diluted\\n$ (6.81) $ (3.86) $ (0.29) Weighted-average shares used to compute net loss per shar\\ne attributable to common stockholders:Basic\\n1,248,353 1,752,960 1,892,546 Diluted\\n1,248,353 1,752,960 1,895,519 The accompanying notes are an integral part of \\nthese consolidated financial statements.75', start_char_idx=0, end_char_idx=1560, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=0.8510830998420715)]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Performing RAG with Metadata Filtering\n",
        "\n",
        "Each LlamaIndex document contains a metadata field.\n",
        "The default metadata field contains following fields\n",
        "\n",
        "```\n",
        "metadata={'page_label': '129', 'file_name': 'uber_2021.pdf', 'file_path': 'data/10k/uber_2021.pdf', 'file_type': 'application/pdf', 'file_size': 1880483, 'creation_date': '2024-02-16', 'last_modified_date': '2024-02-16', 'last_accessed_date': '2024-02-16'}\n",
        "```\n",
        "\n",
        "\n",
        "The metadata can be customized using instructions at\n",
        "https://docs.llamaindex.ai/en/stable/module_guides/loading/documents_and_nodes/usage_documents.html\n",
        "\n",
        "\n",
        "Whatever metadata field is desired to be searchable, needs to be included in the Atlas Vector Search index definition. In the example above we have included the field `metadata.page_label` in the Vector Search index definition via the `type: filter`\n",
        "```\n",
        "{\n",
        "      \"type\": \"filter\",\n",
        "      \"path\": \"metadata.page_label\"\n",
        "    }\n",
        "```\n",
        "\n",
        "In the above response, we can see that the answer was contained in the page_label = 129.\n",
        "\n",
        "If we exclude that particular page_label we wont be able to find the corresponding answer"
      ],
      "metadata": {
        "id": "u-gENDA_LjKf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.core.vector_stores import (\n",
        "    MetadataFilter,\n",
        "    MetadataFilters,\n",
        "    ExactMatchFilter,\n",
        "    FilterOperator,\n",
        ")\n",
        "filters = MetadataFilters(\n",
        "    filters=[ExactMatchFilter(key=\"metadata.page_label\", value=\"131\")]\n",
        ")\n",
        "response = index.as_query_engine(filters=filters).query(\"What was Uber's revenue?\", )\n",
        "print(response)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zTh66z7aGQgs",
        "outputId": "6df85507-a5a2-4802-9605-9afe11e335f1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "I'm sorry, but the given context does not provide any information about Uber's revenue.\n"
          ]
        }
      ]
    }
  ]
}