achatbot_deepseekVL2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "L4",
      "authorship_tag": "ABX9TyP/JqLYzHWoukxltuNGW0xL",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/weedge/doraemon-nb/blob/main/achatbot_deepseekVL2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# install"
      ],
      "metadata": {
        "id": "-DSxG0KC86f9"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CepHVoTB7OCO"
      },
      "outputs": [],
      "source": [
        "!cd /content && rm -rf achatbot && git clone --recursive https://github.com/ai-bot-pro/achatbot.git"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd /content/achatbot"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P1i4Vmbr712c",
        "outputId": "48d3a85c-d181-4142-d995-b8b0ecfe83ff"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/achatbot\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!bash scripts/pypi_achatbot.sh dev\n"
      ],
      "metadata": {
        "id": "K5zg_Mi278Aj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q \"dist/achatbot-0.0.8.7-py3-none-any.whl[llm_transformers_manual_vision_deepseekvl2]\""
      ],
      "metadata": {
        "id": "1R-SA5Ci78xD"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip show transformers"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_z3BLBrCjlNi",
        "outputId": "c2c093af-06f2-4631-8f4d-6a62d00a1f18"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Name: transformers\n",
            "Version: 4.38.2\n",
            "Summary: State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow\n",
            "Home-page: https://github.com/huggingface/transformers\n",
            "Author: The Hugging Face team (past and future) with the help of all our contributors (https://github.com/huggingface/transformers/graphs/contributors)\n",
            "Author-email: transformers@huggingface.co\n",
            "License: Apache 2.0 License\n",
            "Location: /usr/local/lib/python3.11/dist-packages\n",
            "Requires: filelock, huggingface-hub, numpy, packaging, pyyaml, regex, requests, safetensors, tokenizers, tqdm\n",
            "Required-by: peft, sentence-transformers\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# download model\n"
      ],
      "metadata": {
        "id": "u0ShW0538J2y"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "Oh9kyx4Dg8ZU"
      },
      "outputs": [],
      "source": [
        "from google.colab import userdata\n",
        "HF_TOKEN=userdata.get('HF_TOKEN')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jGGYbtpNgSq4"
      },
      "outputs": [],
      "source": [
        "!huggingface-cli login --token $HF_TOKEN --add-to-git-credential"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!huggingface-cli download deepseek-ai/deepseek-vl2-tiny --quiet --local-dir /content/models/deepseek-ai/deepseek-vl2-tiny"
      ],
      "metadata": {
        "id": "XViIJ-jqRAr1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b1139e17-9b2c-4cf4-89f6-0b7a6e554f40"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/models/deepseek-ai/deepseek-vl2-tiny\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!huggingface-cli download deepseek-ai/deepseek-vl2-small --quiet --local-dir /content/models/deepseek-ai/deepseek-vl2-small"
      ],
      "metadata": {
        "id": "EoDnwRHNRIV3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!huggingface-cli download deepseek-ai/deepseek-vl2 --quiet --local-dir /content/models/deepseek-ai/deepseek-vl2"
      ],
      "metadata": {
        "id": "7JSOq77IRI15"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# demo"
      ],
      "metadata": {
        "id": "3iEI0eyPUpQz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!mkdir -p /content/images/\n",
        "!wget https://github.com/deepseek-ai/DeepSeek-VL2/blob/main/images/visual_grounding_1.jpeg?raw=true -O /content/images/visual_grounding.jpeg"
      ],
      "metadata": {
        "id": "3rPiWUBcV0Pl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "\n",
        "def print_model_params(model: torch.nn.Module, extra_info=\"\"):\n",
        "    # print the number of parameters in the model\n",
        "    model_million_params = sum(p.numel() for p in model.parameters()) / 1e6\n",
        "    print(model)\n",
        "    print(f\"{extra_info} {model_million_params} M parameters\")"
      ],
      "metadata": {
        "id": "FQQ8ZFYYZhHT"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import sys\n",
        "import os\n",
        "import torch\n",
        "from transformers import AutoModelForCausalLM\n",
        "\n",
        "#os.environ['XFORMERS_DISABLED'] = '1'  # Disable xFormers for this run\n",
        "#os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments'\n",
        "\n",
        "sys.path.insert(1, \"/content/achatbot/deps/DeepSeekVL2\")\n",
        "from deepseek_vl2.models import DeepseekVLV2Processor, DeepseekVLV2ForCausalLM\n",
        "from deepseek_vl2.utils.io import load_pil_images\n",
        "\n",
        "\n",
        "# specify the path to the model\n",
        "model_path = \"/content/models/deepseek-ai/deepseek-vl2-tiny\"\n",
        "vl_chat_processor: DeepseekVLV2Processor = DeepseekVLV2Processor.from_pretrained(model_path)\n",
        "tokenizer = vl_chat_processor.tokenizer\n",
        "\n",
        "vl_gpt: DeepseekVLV2ForCausalLM = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True)\n",
        "vl_gpt = vl_gpt.to(torch.bfloat16).cuda().eval()\n",
        "print_model_params(vl_gpt)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uaZMOEdjVK2Y",
        "outputId": "d48e328e-5322-4676-d347-8af2577e679d"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Python version is above 3.10, patching the collections module.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Add pad token = ['<｜▁pad▁｜>'] to the tokenizer\n",
            "<｜▁pad▁｜>:2\n",
            "Add image token = ['<image>'] to the tokenizer\n",
            "<image>:128815\n",
            "Add grounding-related tokens = ['<|ref|>', '<|/ref|>', '<|det|>', '<|/det|>', '<|grounding|>'] to the tokenizer with input_ids\n",
            "<|ref|>:128816\n",
            "<|/ref|>:128817\n",
            "<|det|>:128818\n",
            "<|/det|>:128819\n",
            "<|grounding|>:128820\n",
            "Add chat tokens = ['<|User|>', '<|Assistant|>'] to the tokenizer with input_ids\n",
            "<|User|>:128821\n",
            "<|Assistant|>:128822\n",
            "\n",
            "DeepseekVLV2ForCausalLM(\n",
            "  (vision): VisionTransformer(\n",
            "    (patch_embed): PatchEmbed(\n",
            "      (proj): Conv2d(3, 1152, kernel_size=(14, 14), stride=(14, 14))\n",
            "      (norm): Identity()\n",
            "    )\n",
            "    (pos_drop): Dropout(p=0.0, inplace=False)\n",
            "    (patch_drop): Identity()\n",
            "    (norm_pre): Identity()\n",
            "    (blocks): Sequential(\n",
            "      (0): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (1): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (2): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (3): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (4): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (5): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (6): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (7): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (8): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (9): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (10): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (11): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (12): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (13): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (14): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (15): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (16): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (17): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (18): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (19): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (20): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (21): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (22): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (23): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (24): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (25): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (26): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "    )\n",
            "    (norm): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "    (attn_pool): AttentionPoolLatent(\n",
            "      (q): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "      (kv): Linear(in_features=1152, out_features=2304, bias=True)\n",
            "      (q_norm): Identity()\n",
            "      (k_norm): Identity()\n",
            "      (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "      (proj_drop): Dropout(p=0.0, inplace=False)\n",
            "      (norm): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "      (mlp): Mlp(\n",
            "        (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "        (act): GELU(approximate='none')\n",
            "        (drop1): Dropout(p=0.0, inplace=False)\n",
            "        (norm): Identity()\n",
            "        (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "        (drop2): Dropout(p=0.0, inplace=False)\n",
            "      )\n",
            "    )\n",
            "    (fc_norm): Identity()\n",
            "    (head_drop): Dropout(p=0.0, inplace=False)\n",
            "    (head): Identity()\n",
            "  )\n",
            "  (projector): MlpProjector(\n",
            "    (layers): Sequential(\n",
            "      (0): Linear(in_features=4608, out_features=1280, bias=True)\n",
            "      (1): GELU(approximate='none')\n",
            "      (2): Linear(in_features=1280, out_features=1280, bias=True)\n",
            "    )\n",
            "  )\n",
            "  (language): DeepseekV2ForCausalLM(\n",
            "    (model): DeepseekV2Model(\n",
            "      (embed_tokens): Embedding(129280, 1280)\n",
            "      (layers): ModuleList(\n",
            "        (0): DeepseekV2DecoderLayer(\n",
            "          (self_attn): LlamaAttention(\n",
            "            (q_proj): Linear(in_features=1280, out_features=1280, bias=False)\n",
            "            (k_proj): Linear(in_features=1280, out_features=1280, bias=False)\n",
            "            (v_proj): Linear(in_features=1280, out_features=1280, bias=False)\n",
            "            (o_proj): Linear(in_features=1280, out_features=1280, bias=False)\n",
            "            (rotary_emb): LlamaRotaryEmbedding()\n",
            "          )\n",
            "          (mlp): DeepseekV2MLP(\n",
            "            (gate_proj): Linear(in_features=1280, out_features=6848, bias=False)\n",
            "            (up_proj): Linear(in_features=1280, out_features=6848, bias=False)\n",
            "            (down_proj): Linear(in_features=6848, out_features=1280, bias=False)\n",
            "            (act_fn): SiLU()\n",
            "          )\n",
            "          (input_layernorm): DeepseekV2RMSNorm()\n",
            "          (post_attention_layernorm): DeepseekV2RMSNorm()\n",
            "        )\n",
            "        (1-11): 11 x DeepseekV2DecoderLayer(\n",
            "          (self_attn): LlamaAttention(\n",
            "            (q_proj): Linear(in_features=1280, out_features=1280, bias=False)\n",
            "            (k_proj): Linear(in_features=1280, out_features=1280, bias=False)\n",
            "            (v_proj): Linear(in_features=1280, out_features=1280, bias=False)\n",
            "            (o_proj): Linear(in_features=1280, out_features=1280, bias=False)\n",
            "            (rotary_emb): LlamaRotaryEmbedding()\n",
            "          )\n",
            "          (mlp): DeepseekV2MoE(\n",
            "            (experts): ModuleList(\n",
            "              (0-63): 64 x DeepseekV2MLP(\n",
            "                (gate_proj): Linear(in_features=1280, out_features=896, bias=False)\n",
            "                (up_proj): Linear(in_features=1280, out_features=896, bias=False)\n",
            "                (down_proj): Linear(in_features=896, out_features=1280, bias=False)\n",
            "                (act_fn): SiLU()\n",
            "              )\n",
            "            )\n",
            "            (gate): MoEGate()\n",
            "            (shared_experts): DeepseekV2MLP(\n",
            "              (gate_proj): Linear(in_features=1280, out_features=1792, bias=False)\n",
            "              (up_proj): Linear(in_features=1280, out_features=1792, bias=False)\n",
            "              (down_proj): Linear(in_features=1792, out_features=1280, bias=False)\n",
            "              (act_fn): SiLU()\n",
            "            )\n",
            "          )\n",
            "          (input_layernorm): DeepseekV2RMSNorm()\n",
            "          (post_attention_layernorm): DeepseekV2RMSNorm()\n",
            "        )\n",
            "      )\n",
            "      (norm): DeepseekV2RMSNorm()\n",
            "    )\n",
            "    (lm_head): Linear(in_features=1280, out_features=129280, bias=False)\n",
            "  )\n",
            ")\n",
            " 3370.50144 M parameters\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## single image conversation example\n",
        "\n",
        "conversation = [\n",
        "    {\n",
        "        \"role\": \"User\",\n",
        "        \"content\": \"<image>\\n<|ref|>The giraffe at the back.<|/ref|>.\",\n",
        "        \"images\": [\"/content/images/visual_grounding.jpeg\"],\n",
        "    },\n",
        "    {\"role\": \"<|Assistant|>\", \"content\": \"\"},\n",
        "]\n",
        "\n",
        "## multiple images (or in-context learning) conversation example\n",
        "# conversation = [\n",
        "#     {\n",
        "#         \"role\": \"User\",\n",
        "#         \"content\": \"<image_placeholder>A dog wearing nothing in the foreground, \"\n",
        "#                    \"<image_placeholder>a dog wearing a santa hat, \"\n",
        "#                    \"<image_placeholder>a dog wearing a wizard outfit, and \"\n",
        "#                    \"<image_placeholder>what's the dog wearing?\",\n",
        "#         \"images\": [\n",
        "#             \"images/dog_a.png\",\n",
        "#             \"images/dog_b.png\",\n",
        "#             \"images/dog_c.png\",\n",
        "#             \"images/dog_d.png\",\n",
        "#         ],\n",
        "#     },\n",
        "#     {\"role\": \"Assistant\", \"content\": \"\"}\n",
        "# ]\n",
        "\n",
        "# load images and prepare for inputs\n",
        "pil_images = load_pil_images(conversation)\n",
        "prepare_inputs = vl_chat_processor(\n",
        "    conversations=conversation,\n",
        "    images=pil_images,\n",
        "    force_batchify=True,\n",
        "    system_prompt=\"\"\n",
        ").to(vl_gpt.device, dtype=torch.bfloat16)\n",
        "#print(prepare_inputs)\n",
        "# run image encoder to get the image embeddings\n",
        "inputs_embeds = vl_gpt.prepare_inputs_embeds(**prepare_inputs)\n",
        "\n",
        "# run the model to get the response\n",
        "outputs = vl_gpt.language.generate(\n",
        "    #input_ids = prepare_inputs[\"input_ids\"],\n",
        "    inputs_embeds=inputs_embeds,\n",
        "    attention_mask=prepare_inputs.attention_mask,\n",
        "    pad_token_id=tokenizer.eos_token_id,\n",
        "    bos_token_id=tokenizer.bos_token_id,\n",
        "    eos_token_id=tokenizer.eos_token_id,\n",
        "    max_new_tokens=512,\n",
        "    do_sample=False,\n",
        "    use_cache=True,\n",
        "    #position_ids=torch.arange(inputs_embeds.shape[1], device=vl_gpt.device).unsqueeze(0)\n",
        ")\n",
        "\n",
        "answer = tokenizer.decode(outputs[0].cpu().tolist(), skip_special_tokens=True)\n",
        "print(f\"{prepare_inputs['sft_format'][0]}\", answer)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NHxrWBoPUqWh",
        "outputId": "79c4c225-4b3c-435e-a218-7fed322fae82"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "You're using a LlamaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<|User|>: <image>\n",
            "<|ref|>The giraffe at the back.<|/ref|>.\n",
            "\n",
            "<|Assistant|>: The giraffe at the back.[[580, 270, 999, 900]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# test"
      ],
      "metadata": {
        "id": "bziiC0sPROau"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!LLM_DEVICE=cuda LLM_MODEL_NAME_OR_PATH=/content/models/deepseek-ai/deepseek-vl2-tiny \\\n",
        "    python -m unittest test.core.llm.test_transformers_v_deepseek.TestTransformersVJanus.test_chat_completion_prompts"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ds1gTmkqRP5g",
        "outputId": "0fd493cb-4679-4a05-e1b3-09223cd6b3f0"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Python version is above 3.10, patching the collections module.\n",
            "2025-02-06 14:30:18,071 - git.cmd - DEBUG - /usr/local/lib/python3.11/dist-packages/git/cmd.py:1253 - execute - Popen(['git', 'version'], cwd=/content/achatbot, stdin=None, shell=False, universal_newlines=False)\n",
            "2025-02-06 14:30:18,073 - git.cmd - DEBUG - /usr/local/lib/python3.11/dist-packages/git/cmd.py:1253 - execute - Popen(['git', 'version'], cwd=/content/achatbot, stdin=None, shell=False, universal_newlines=False)\n",
            "2025-02-06 14:30:18,116 - wandb.docker.auth - DEBUG - /usr/local/lib/python3.11/dist-packages/wandb/docker/auth.py:50 - find_config_file - Trying paths: ['/root/.docker/config.json', '/root/.dockercfg']\n",
            "2025-02-06 14:30:18,117 - wandb.docker.auth - DEBUG - /usr/local/lib/python3.11/dist-packages/wandb/docker/auth.py:57 - find_config_file - No config file found\n",
            "2025-02-06 14:30:19,005 - chat-bot - INFO - /content/achatbot/src/common/factory.py:69 - get_engine_by_tag - use llm_transformers_manual_vision_deepseek engine\n",
            "2025-02-06 14:30:19,005 - chat-bot - INFO - /content/achatbot/src/common/factory.py:34 - get_instance - class: <class 'src.core.llm.transformers.manual_vision_deepseek.TransformersManualVisionDeepSeekVL2'> args: {'lm_model_name_or_path': '/content/models/deepseek-ai/deepseek-vl2-tiny', 'lm_device_map': None, 'lm_device': 'cuda', 'lm_torch_dtype': 'auto', 'lm_attn_impl': None, 'user_role': 'user', 'warnup_prompt': \"Repeat the word 'weedge niu bi'.\", 'warnup_steps': 2, 'init_chat_role': 'system', 'init_chat_prompt': '', 'lm_gen_seed': 42, 'lm_gen_max_new_tokens': 1024, 'lm_gen_min_new_tokens': 0, 'lm_gen_do_sample': False, 'lm_gen_temperature': 0.0, 'lm_gen_top_k': 1, 'lm_gen_top_p': 0.8, 'lm_gen_repetition_penalty': 1.0, 'lm_tokenizer_decode_batch_size': 60, 'chat_history_size': 10, 'lm_stream': True, 'model_type': 'chat_completion', 'lm_bnb_quant_type': 'int4'}\n",
            "2025-02-06 14:30:19,005 - chat-bot - INFO - /content/achatbot/src/core/llm/transformers/manual_vision_deepseek.py:50 - __init__ - TransformersLMArgs: TransformersLMArgs(lm_model_name_or_path='/content/models/deepseek-ai/deepseek-vl2-tiny', lm_device_map=None, lm_device='cuda', lm_torch_dtype='auto', lm_attn_impl=None, user_role='user', warnup_prompt=\"Repeat the word 'weedge niu bi'.\", warnup_steps=2, init_chat_role='system', init_chat_prompt='', lm_gen_seed=42, lm_gen_max_new_tokens=1024, lm_gen_min_new_tokens=0, lm_gen_do_sample=False, lm_gen_temperature=0.0, lm_gen_top_k=1, lm_gen_top_p=0.8, lm_gen_repetition_penalty=1.0, lm_tokenizer_decode_batch_size=60, chat_history_size=10, lm_stream=True, model_type='chat_completion', lm_bnb_quant_type='int4')\n",
            "2025-02-06 14:30:25,116 - chat-bot - DEBUG - /content/achatbot/src/common/utils/helper.py:54 - print_model_params - DeepseekVLV2ForCausalLM(\n",
            "  (vision): VisionTransformer(\n",
            "    (patch_embed): PatchEmbed(\n",
            "      (proj): Conv2d(3, 1152, kernel_size=(14, 14), stride=(14, 14))\n",
            "      (norm): Identity()\n",
            "    )\n",
            "    (pos_drop): Dropout(p=0.0, inplace=False)\n",
            "    (patch_drop): Identity()\n",
            "    (norm_pre): Identity()\n",
            "    (blocks): Sequential(\n",
            "      (0): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (1): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (2): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (3): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (4): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (5): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (6): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (7): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (8): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (9): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (10): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (11): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (12): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (13): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (14): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (15): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (16): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (17): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (18): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (19): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (20): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (21): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (22): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (23): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (24): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (25): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "      (26): Block(\n",
            "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (attn): Attention(\n",
            "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
            "          (q_norm): Identity()\n",
            "          (k_norm): Identity()\n",
            "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
            "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "          (proj_drop): Identity()\n",
            "        )\n",
            "        (ls1): Identity()\n",
            "        (drop_path1): Identity()\n",
            "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "        (mlp): Mlp(\n",
            "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "          (act): GELU(approximate='tanh')\n",
            "          (drop1): Dropout(p=0.0, inplace=False)\n",
            "          (norm): Identity()\n",
            "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "          (drop2): Dropout(p=0.0, inplace=False)\n",
            "        )\n",
            "        (ls2): Identity()\n",
            "        (drop_path2): Identity()\n",
            "      )\n",
            "    )\n",
            "    (norm): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "    (attn_pool): AttentionPoolLatent(\n",
            "      (q): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "      (kv): Linear(in_features=1152, out_features=2304, bias=True)\n",
            "      (q_norm): Identity()\n",
            "      (k_norm): Identity()\n",
            "      (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
            "      (proj_drop): Dropout(p=0.0, inplace=False)\n",
            "      (norm): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
            "      (mlp): Mlp(\n",
            "        (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
            "        (act): GELU(approximate='none')\n",
            "        (drop1): Dropout(p=0.0, inplace=False)\n",
            "        (norm): Identity()\n",
            "        (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
            "        (drop2): Dropout(p=0.0, inplace=False)\n",
            "      )\n",
            "    )\n",
            "    (fc_norm): Identity()\n",
            "    (head_drop): Dropout(p=0.0, inplace=False)\n",
            "    (head): Identity()\n",
            "  )\n",
            "  (projector): MlpProjector(\n",
            "    (layers): Sequential(\n",
            "      (0): Linear(in_features=4608, out_features=1280, bias=True)\n",
            "      (1): GELU(approximate='none')\n",
            "      (2): Linear(in_features=1280, out_features=1280, bias=True)\n",
            "    )\n",
            "  )\n",
            "  (language): DeepseekV2ForCausalLM(\n",
            "    (model): DeepseekV2Model(\n",
            "      (embed_tokens): Embedding(129280, 1280)\n",
            "      (layers): ModuleList(\n",
            "        (0): DeepseekV2DecoderLayer(\n",
            "          (self_attn): LlamaAttention(\n",
            "            (q_proj): Linear(in_features=1280, out_features=1280, bias=False)\n",
            "            (k_proj): Linear(in_features=1280, out_features=1280, bias=False)\n",
            "            (v_proj): Linear(in_features=1280, out_features=1280, bias=False)\n",
            "            (o_proj): Linear(in_features=1280, out_features=1280, bias=False)\n",
            "            (rotary_emb): LlamaRotaryEmbedding()\n",
            "          )\n",
            "          (mlp): DeepseekV2MLP(\n",
            "            (gate_proj): Linear(in_features=1280, out_features=6848, bias=False)\n",
            "            (up_proj): Linear(in_features=1280, out_features=6848, bias=False)\n",
            "            (down_proj): Linear(in_features=6848, out_features=1280, bias=False)\n",
            "            (act_fn): SiLU()\n",
            "          )\n",
            "          (input_layernorm): DeepseekV2RMSNorm()\n",
            "          (post_attention_layernorm): DeepseekV2RMSNorm()\n",
            "        )\n",
            "        (1-11): 11 x DeepseekV2DecoderLayer(\n",
            "          (self_attn): LlamaAttention(\n",
            "            (q_proj): Linear(in_features=1280, out_features=1280, bias=False)\n",
            "            (k_proj): Linear(in_features=1280, out_features=1280, bias=False)\n",
            "            (v_proj): Linear(in_features=1280, out_features=1280, bias=False)\n",
            "            (o_proj): Linear(in_features=1280, out_features=1280, bias=False)\n",
            "            (rotary_emb): LlamaRotaryEmbedding()\n",
            "          )\n",
            "          (mlp): DeepseekV2MoE(\n",
            "            (experts): ModuleList(\n",
            "              (0-63): 64 x DeepseekV2MLP(\n",
            "                (gate_proj): Linear(in_features=1280, out_features=896, bias=False)\n",
            "                (up_proj): Linear(in_features=1280, out_features=896, bias=False)\n",
            "                (down_proj): Linear(in_features=896, out_features=1280, bias=False)\n",
            "                (act_fn): SiLU()\n",
            "              )\n",
            "            )\n",
            "            (gate): MoEGate()\n",
            "            (shared_experts): DeepseekV2MLP(\n",
            "              (gate_proj): Linear(in_features=1280, out_features=1792, bias=False)\n",
            "              (up_proj): Linear(in_features=1280, out_features=1792, bias=False)\n",
            "              (down_proj): Linear(in_features=1792, out_features=1280, bias=False)\n",
            "              (act_fn): SiLU()\n",
            "            )\n",
            "          )\n",
            "          (input_layernorm): DeepseekV2RMSNorm()\n",
            "          (post_attention_layernorm): DeepseekV2RMSNorm()\n",
            "        )\n",
            "      )\n",
            "      (norm): DeepseekV2RMSNorm()\n",
            "    )\n",
            "    (lm_head): Linear(in_features=1280, out_features=129280, bias=False)\n",
            "  )\n",
            ")\n",
            "2025-02-06 14:30:25,226 - chat-bot - DEBUG - /content/achatbot/src/common/utils/helper.py:55 - print_model_params - llm_transformers_manual_vision_deepseek 3370.50144 M parameters\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
            "Add pad token = ['<｜▁pad▁｜>'] to the tokenizer\n",
            "<｜▁pad▁｜>:2\n",
            "Add image token = ['<image>'] to the tokenizer\n",
            "<image>:128815\n",
            "Add grounding-related tokens = ['<|ref|>', '<|/ref|>', '<|det|>', '<|/det|>', '<|grounding|>'] to the tokenizer with input_ids\n",
            "<|ref|>:128816\n",
            "<|/ref|>:128817\n",
            "<|det|>:128818\n",
            "<|/det|>:128819\n",
            "<|grounding|>:128820\n",
            "Add chat tokens = ['<|User|>', '<|Assistant|>'] to the tokenizer with input_ids\n",
            "<|User|>:128821\n",
            "<|Assistant|>:128822\n",
            "\n",
            "You're using a LlamaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
            "2025-02-06 14:30:30.309905: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:477] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
            "E0000 00:00:1738852230.328469   24969 cuda_dnn.cc:8310] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "E0000 00:00:1738852230.334016   24969 cuda_blas.cc:1418] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2025-02-06 14:30:31,313 - tensorflow - DEBUG - /usr/local/lib/python3.11/dist-packages/tensorflow/python/distribute/cluster_resolver/tpu/tpu_cluster_resolver.py:34 - <module> - Falling back to TensorFlow client; we recommended you install the Cloud TPU client directly with pip install cloud-tpu-client.\n",
            "2025-02-06 14:30:31,481 - h5py._conv - DEBUG - /usr/local/lib/python3.11/dist-packages/h5py/__init__.py:47 - <module> - Creating converter from 7 to 5\n",
            "2025-02-06 14:30:31,481 - h5py._conv - DEBUG - /usr/local/lib/python3.11/dist-packages/h5py/__init__.py:47 - <module> - Creating converter from 5 to 7\n",
            "2025-02-06 14:30:31,481 - h5py._conv - DEBUG - /usr/local/lib/python3.11/dist-packages/h5py/__init__.py:47 - <module> - Creating converter from 7 to 5\n",
            "2025-02-06 14:30:31,481 - h5py._conv - DEBUG - /usr/local/lib/python3.11/dist-packages/h5py/__init__.py:47 - <module> - Creating converter from 5 to 7\n",
            "2025-02-06 14:30:32,024 - numexpr.utils - INFO - /usr/local/lib/python3.11/dist-packages/numexpr/utils.py:162 - _init_num_threads - NumExpr defaulting to 12 threads.\n",
            "2025-02-06 14:30:32,242 - matplotlib - DEBUG - /usr/local/lib/python3.11/dist-packages/matplotlib/__init__.py:342 - wrapper - matplotlib data path: /usr/local/lib/python3.11/dist-packages/matplotlib/mpl-data\n",
            "2025-02-06 14:30:32,251 - matplotlib - DEBUG - /usr/local/lib/python3.11/dist-packages/matplotlib/__init__.py:342 - wrapper - CONFIGDIR=/root/.config/matplotlib\n",
            "2025-02-06 14:30:32,253 - matplotlib - DEBUG - /usr/local/lib/python3.11/dist-packages/matplotlib/__init__.py:1557 - <module> - interactive is False\n",
            "2025-02-06 14:30:32,254 - matplotlib - DEBUG - /usr/local/lib/python3.11/dist-packages/matplotlib/__init__.py:1558 - <module> - platform is linux\n",
            "2025-02-06 14:30:32,579 - matplotlib - DEBUG - /usr/local/lib/python3.11/dist-packages/matplotlib/__init__.py:342 - wrapper - CACHEDIR=/root/.cache/matplotlib\n",
            "2025-02-06 14:30:32,580 - matplotlib.font_manager - DEBUG - /usr/local/lib/python3.11/dist-packages/matplotlib/font_manager.py:1635 - _load_fontmanager - Using fontManager instance from /root/.cache/matplotlib/fontlist-v390.json\n",
            "2025-02-06 14:30:33,289 - chat-bot - INFO - /content/achatbot/src/core/llm/transformers/base.py:134 - _warmup - Warming up TransformersManualVisionDeepSeekVL2 device: cuda:0\n",
            "/usr/local/lib/python3.11/dist-packages/transformers/generation/configuration_utils.py:410: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.0` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.11/dist-packages/transformers/generation/configuration_utils.py:415: UserWarning: `do_sample` is set to `False`. However, `top_p` is set to `0.8` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_p`.\n",
            "  warnings.warn(\n",
            "2025-02-06 14:30:33,879 - chat-bot - INFO - /content/achatbot/src/core/llm/transformers/base.py:152 - _warmup - step 0 warnup TTFT time: 0.3303783139999723 s\n",
            "2025-02-06 14:30:34,227 - chat-bot - INFO - /content/achatbot/src/core/llm/transformers/base.py:152 - _warmup - step 1 warnup TTFT time: 0.11392121599965321 s\n",
            "2025-02-06 14:30:34,227 - chat-bot - INFO - /content/achatbot/src/core/llm/transformers/base.py:157 - _warmup - TransformersManualVisionDeepSeekVL2:  warmed up! time: 0.938 s\n",
            "2025-02-06 14:30:34,227 - chat-bot - INFO - /content/achatbot/src/core/llm/__init__.py:54 - initLLMEngine - initLLMEngine: llm_transformers_manual_vision_deepseek, TAG:llm_transformers_manual_vision_deepseek | TransformersManualVisionDeepSeekVL2\n",
            "\n",
            "--------test prompt:  [<PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=570x380 at 0x795E95AAEAD0>, '描述下图片中的内容'] --------\n",
            "\n",
            "2025-02-06 14:30:34,232 - chat-bot - DEBUG - /content/achatbot/test/core/llm/test_transformers_v_deepseek.py:61 - test_chat_completion_prompts - session ctx: {'client_id': 'test_llm_transformers_manual_vision_deepseek_client_id', 'sampling_rate': 16000, 'sample_width': 2, 'read_audio_frames_len': 0, 'state': {'prompt': [<PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=570x380 at 0x795E95AAEAD0>, '描述下图片中的内容']}}\n",
            "2025-02-06 14:30:34,232 - chat-bot - DEBUG - /content/achatbot/test/core/llm/test_transformers_v_deepseek.py:62 - test_chat_completion_prompts - TransformersLMArgs(lm_model_name_or_path='/content/models/deepseek-ai/deepseek-vl2-tiny', lm_device_map=None, lm_device='cuda', lm_torch_dtype='auto', lm_attn_impl=None, user_role='user', warnup_prompt=\"Repeat the word 'weedge niu bi'.\", warnup_steps=2, init_chat_role='system', init_chat_prompt='', lm_gen_seed=42, lm_gen_max_new_tokens=1024, lm_gen_min_new_tokens=0, lm_gen_do_sample=False, lm_gen_temperature=0.0, lm_gen_top_k=1, lm_gen_top_p=0.8, lm_gen_repetition_penalty=1.0, lm_tokenizer_decode_batch_size=60, chat_history_size=10, lm_stream=True, model_type='chat_completion', lm_bnb_quant_type='int4')\n",
            "2025-02-06 14:30:34,232 - chat-bot - DEBUG - /content/achatbot/src/core/llm/transformers/manual_vision_deepseek.py:129 - generate - kwargs: {}\n",
            "2025-02-06 14:30:34,243 - chat-bot - DEBUG - /content/achatbot/src/core/llm/transformers/manual_vision_deepseek.py:152 - generate - chat_history:[{'role': '<|User|>', 'content': '<image>\\n描述下图片中的内容'}]\n",
            "2025-02-06 14:30:37,150 - chat-bot - DEBUG - /content/achatbot/test/core/llm/test_transformers_v_deepseek.py:73 - test_chat_completion_prompts - chat_completion TTFT time: 0.4922861039995041 s\n",
            "2025-02-06 14:30:37,150 - chat-bot - DEBUG - /content/achatbot/test/core/llm/test_transformers_v_deepseek.py:74 - test_chat_completion_prompts - generated text: 图片中展示了一位穿着黄色上衣的男子，他正在城市街道上熨衣服。他站在一辆黄色的SUV车尾，车尾上安装了一个熨衣架，上面放着一件蓝色的衣物。背景中可以看到城市街道，有黄色的出租车和其他车辆，以及一些高楼建筑。街道两旁悬挂着粉红色的旗帜，上面似乎有一些图案或文字。整幅图像给人一种忙碌的城市生活氛围。\n",
            ".\n",
            "----------------------------------------------------------------------\n",
            "Ran 1 test in 20.934s\n",
            "\n",
            "OK\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 总结\n",
        "- 虽然MoE初始化占用的GPU内存大，但是推理运行时，激活参数所用的GPU内存少（相对稠密型）\n",
        "- 具体部署体验见： https://github.com/ai-bot-pro/achatbot/pull/114 这个pr进行部署"
      ],
      "metadata": {
        "id": "TwI7vWmplUAm"
      }
    }
  ]
}