example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use MEt3R with DUSt3R weights from MASt3R "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /BS/grl-masim-data/work/torch_models/hub/mhamilton723_FeatUp_main\n",
      "/BS/grl-co3d/work/install/miniconda3/envs/test/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "Using cache found in /BS/grl-masim-data/work/torch_models/hub/facebookresearch_dino_main\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from met3r import MEt3R\n",
    "\n",
    "IMG_SIZE = 256\n",
    "\n",
    "# Initialize MEt3R\n",
    "metric = MEt3R(\n",
    "    img_size=IMG_SIZE, # Default. Set to `None` to use the input resolution on the fly!\n",
    "    use_norm=True, # Default \n",
    "    feat_backbone=\"dino16\", # Default \n",
    "    featup_weights=\"mhamilton723/FeatUp\", # Default \n",
    "    dust3r_weights=\"naver/MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric\", # Default\n",
    "    use_mast3r_dust3r=True # Default. Set to `False` to use original DUSt3R. Make sure to also set the correct weights from huggingface.\n",
    ").cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/BS/grl-co3d/work/install/miniconda3/envs/test/lib/python3.10/site-packages/torch/functional.py:512: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3587.)\n",
      "  return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.28395265340805054\n"
     ]
    }
   ],
   "source": [
    "# Prepare inputs of shape (batch, views, channels, height, width): views must be 2\n",
    "# RGB range must be in [-1, 1]\n",
    "# Reduce the batch size in case of CUDA OOM\n",
    "inputs = torch.randn((10, 2, 3, IMG_SIZE, IMG_SIZE)).cuda()\n",
    "inputs = inputs.clip(-1, 1)\n",
    "\n",
    "# Evaluate MEt3R\n",
    "score, *_ = metric(\n",
    "    images=inputs, \n",
    "    return_overlap_mask=False, # Default \n",
    "    return_score_map=False, # Default \n",
    "    return_projections=False # Default \n",
    ")\n",
    "\n",
    "# Should be between 0.25 - 0.35\n",
    "print(score.mean().item())\n",
    "\n",
    "# Clear up GPU memory\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use MEt3R with original DUSt3R weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /BS/grl-masim-data/work/torch_models/hub/mhamilton723_FeatUp_main\n",
      "Using cache found in /BS/grl-masim-data/work/torch_models/hub/facebookresearch_dino_main\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from met3r import MEt3R\n",
    "\n",
    "IMG_SIZE = 256\n",
    "\n",
    "# Initialize MEt3R\n",
    "metric = MEt3R(\n",
    "    img_size=IMG_SIZE, # Default. Set to `None` to use the input resolution on the fly!\n",
    "    use_norm=True, # Default \n",
    "    feat_backbone=\"dino16\", # Default \n",
    "    featup_weights=\"mhamilton723/FeatUp\", # Default \n",
    "    dust3r_weights=\"naver/DUSt3R_ViTLarge_BaseDecoder_512_dpt\", # Default\n",
    "    use_mast3r_dust3r=False \n",
    ").cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3434220552444458\n"
     ]
    }
   ],
   "source": [
    "# Prepare inputs of shape (batch, views, channels, height, width): views must be 2\n",
    "# RGB range must be in [-1, 1]\n",
    "# Reduce the batch size in case of CUDA OOM\n",
    "inputs = torch.randn((10, 2, 3, IMG_SIZE, IMG_SIZE)).cuda()\n",
    "inputs = inputs.clip(-1, 1)\n",
    "\n",
    "# Evaluate MEt3R\n",
    "score, *_ = metric(\n",
    "    images=inputs, \n",
    "    return_overlap_mask=False, # Default \n",
    "    return_score_map=False, # Default \n",
    "    return_projections=False # Default \n",
    ")\n",
    "\n",
    "# Should be between 0.30 - 0.35\n",
    "print(score.mean().item())\n",
    "\n",
    "# Clear up GPU memory\n",
    "torch.cuda.empty_cache()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "test",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}