achatbot_glm_voice_bot.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/weedge/doraemon-nb/blob/main/achatbot_glm_voice_bot.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-9hXsk51gsUa"
      },
      "source": [
        "# install"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vGsomgs1KFCD"
      },
      "outputs": [],
      "source": [
        "!cd /content && rm -rf achatbot && git clone --recursive https://github.com/ai-bot-pro/achatbot.git"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Kn5eFEj2NuJB"
      },
      "outputs": [],
      "source": [
        "%cd /content/achatbot\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "j9dyrdw-Nx5y"
      },
      "outputs": [],
      "source": [
        "!bash scripts/pypi_achatbot.sh dev"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SIIQ6ULyN3z_"
      },
      "outputs": [],
      "source": [
        "!pip install -q \"dist/achatbot-0.0.7.12-py3-none-any.whl[fastapi_bot_server,daily_room_audio_stream,livekit_room_audio_stream,silero_vad_analyzer,sense_voice_asr,llm_transformers_manual_voice_glm,bitsandbytes,queue]\"\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c4jLuMQhgvO7"
      },
      "source": [
        "# download model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bz6H_bMmjRFr"
      },
      "outputs": [],
      "source": [
        "from google.colab import userdata\n",
        "NGROK_TOKEN=userdata.get('NGROK_TOKEN')\n",
        "HF_TOKEN=userdata.get('HF_TOKEN')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GCGnjWUvitof"
      },
      "outputs": [],
      "source": [
        "!huggingface-cli login --token $HF_TOKEN --add-to-git-credential"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BO_9Lp4DmLZ7"
      },
      "outputs": [],
      "source": [
        "!huggingface-cli download FunAudioLLM/SenseVoiceSmall  --local-dir ./models/FunAudioLLM/SenseVoiceSmall --local-dir-use-symlinks False"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_eEuMHNfjWL5"
      },
      "outputs": [],
      "source": [
        "# GLM-4-Voice-Tokenizer: Trained by adding vector quantization to the encoder part of Whisper,\n",
        "# converting continuous speech input into discrete tokens.\n",
        "# Each second of audio is converted into 12.5 discrete tokens.\n",
        "\n",
        "!huggingface-cli download THUDM/glm-4-voice-tokenizer --local-dir ./models/THUDM/glm-4-voice-tokenizer --local-dir-use-symlinks False"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vHy8iA09-Tox"
      },
      "outputs": [],
      "source": [
        "# GLM-4-Voice-9B: Pre-trained and aligned on speech modality based on GLM-4-9B,\n",
        "# enabling understanding and generation of discretized speech.\n",
        "\n",
        "!huggingface-cli download THUDM/glm-4-voice-9b --local-dir ./models/THUDM/glm-4-voice-9b --local-dir-use-symlinks False"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZxnauK3l0l4X"
      },
      "outputs": [],
      "source": [
        "# GLM-4-Voice-Decoder: A speech decoder supporting streaming inference, retrained based on CosyVoice,\n",
        "# converting discrete speech tokens into continuous speech output.\n",
        "# Generation can start with as few as 10 audio tokens, reducing conversation latency.\n",
        "\n",
        "!git lfs install\n",
        "!git clone https://huggingface.co/THUDM/glm-4-voice-decoder ./models/THUDM/glm-4-voice-decoder"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "heF84BUN8_Ap"
      },
      "outputs": [],
      "source": [
        "#!huggingface-cli download THUDM/glm-4-voice-decoder --local-dir ./models/THUDM/glm-4-voice-decoder --local-dir-use-symlinks False"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# test"
      ],
      "metadata": {
        "id": "JeJXbbZD5OT7"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pM8E941ukgXp"
      },
      "outputs": [],
      "source": [
        "!cd /content/achatbot && python -m unittest test.integration.processors.test_glm_voice_processor.TestProcessor.test_audio\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!cd /content/achatbot && LOG_LEVEL=info GLM_VOICE_TYPE=text python -m unittest test.integration.processors.test_glm_voice_processor.TestProcessor.test_text\n"
      ],
      "metadata": {
        "id": "HeZ65qez2N-J"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fCq_XlOhgytH"
      },
      "source": [
        "# run webrtc bot"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sPhYUWKpk5dD"
      },
      "outputs": [],
      "source": [
        "from google.colab import userdata\n",
        "HF_TOKEN=userdata.get('HF_TOKEN')\n",
        "DAILY_API_KEY=userdata.get('DAILY_API_KEY')\n",
        "\n",
        "LIVEKIT_URL=userdata.get('LIVEKIT_URL')\n",
        "LIVEKIT_API_KEY=userdata.get('LIVEKIT_API_KEY')\n",
        "LIVEKIT_API_SECRET=userdata.get('LIVEKIT_API_SECRET')\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd /content/achatbot"
      ],
      "metadata": {
        "id": "lINDQs9m5uRa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uh5T_mbBk0Qa",
        "outputId": "dfc6fa21-f4eb-46c0-a891-c115116371dc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{\n",
            "  \"chat_bot_name\": \"DailyGLMVoiceBot\",\n",
            "  \"room_name\": \"chat-room\",\n",
            "  \"room_url\": \"\",\n",
            "  \"token\": \"\",\n",
            "  \"room_manager\": {\n",
            "    \"tag\": \"daily_room\",\n",
            "    \"args\": {\n",
            "      \"privacy\": \"public\"\n",
            "    }\n",
            "  },\n",
            "  \"services\": {\n",
            "    \"pipeline\": \"achatbot\",\n",
            "    \"vad\": \"silero\",\n",
            "    \"voice_llm\": \"glm\"\n",
            "  },\n",
            "  \"config\": {\n",
            "    \"vad\": {\n",
            "      \"tag\": \"silero_vad_analyzer\",\n",
            "      \"args\": { \"stop_secs\": 0.7 }\n",
            "    },\n",
            "    \"voice_llm\": {\n",
            "      \"tag\": \"glm_voice_processor\",\n",
            "      \"args\": {\n",
            "        \"lm_gen_args\": {\n",
            "          \"temperature\": 0.2,\n",
            "          \"top_p\": 0.8,\n",
            "          \"max_new_token\": 2000\n",
            "        },\n",
            "        \"voice_out_args\": {\n",
            "          \"audio_sample_rate\": 22050,\n",
            "          \"audio_channels\": 1\n",
            "        },\n",
            "        \"voice_tokenizer_path\": \"./models/THUDM/glm-4-voice-tokenizer\",\n",
            "        \"model_path\": \"./models/THUDM/glm-4-voice-9b\",\n",
            "        \"voice_decoder_path\": \"./models/THUDM/glm-4-voice-decoder\",\n",
            "        \"torch_dtype\": \"auto\",\n",
            "        \"bnb_quant_type\": \"int4\",\n",
            "        \"device\": \"cuda\"\n",
            "      }\n",
            "    }\n",
            "  },\n",
            "  \"config_list\": []\n",
            "}\n"
          ]
        }
      ],
      "source": [
        "!cat /content/daily_glm_voice_bot.json\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wkHdGFOzlAV-"
      },
      "outputs": [],
      "source": [
        "!DAILY_API_KEY=$DAILY_API_KEY \\\n",
        "  python -m src.cmd.bots.main -f /content/daily_glm_voice_bot.json"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {
        "id": "BNZTzBjampd_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f366b23b-9fa7-42f6-fd3b-69f72162419b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{\n",
            "  \"chat_bot_name\": \"DailyAsrGLMVoiceBot\",\n",
            "  \"room_name\": \"chat-room\",\n",
            "  \"room_url\": \"\",\n",
            "  \"token\": \"\",\n",
            "  \"room_manager\": {\n",
            "    \"tag\": \"daily_room\",\n",
            "    \"args\": {\n",
            "      \"privacy\": \"public\"\n",
            "    }\n",
            "  },\n",
            "  \"services\": {\n",
            "    \"pipeline\": \"achatbot\",\n",
            "    \"vad\": \"silero\",\n",
            "    \"asr\": \"sense_voice\",\n",
            "    \"voice_llm\": \"glm\"\n",
            "  },\n",
            "  \"config\": {\n",
            "    \"vad\": {\n",
            "      \"tag\": \"silero_vad_analyzer\",\n",
            "      \"args\": { \"stop_secs\": 0.7 }\n",
            "    },\n",
            "    \"asr\": {\n",
            "      \"tag\": \"sense_voice_asr\",\n",
            "      \"args\": {\n",
            "        \"language\": \"zn\",\n",
            "        \"model_name_or_path\": \"./models/FunAudioLLM/SenseVoiceSmall\"\n",
            "      }\n",
            "    },\n",
            "    \"voice_llm\": {\n",
            "      \"tag\": \"text_glm_voice_processor\",\n",
            "      \"args\": {\n",
            "        \"lm_gen_args\": {\n",
            "          \"temperature\": 0.2,\n",
            "          \"top_p\": 0.8,\n",
            "          \"max_new_token\": 2000\n",
            "        },\n",
            "        \"voice_out_args\": {\n",
            "          \"audio_sample_rate\": 22050,\n",
            "          \"audio_channels\": 1\n",
            "        },\n",
            "        \"voice_tokenizer_path\": \"./models/THUDM/glm-4-voice-tokenizer\",\n",
            "        \"model_path\": \"./models/THUDM/glm-4-voice-9b\",\n",
            "        \"voice_decoder_path\": \"./models/THUDM/glm-4-voice-decoder\",\n",
            "        \"device\": \"cuda\"\n",
            "      }\n",
            "    }\n",
            "  },\n",
            "  \"config_list\": []\n",
            "}\n"
          ]
        }
      ],
      "source": [
        "!cat /content/daily_text_glm_voice_bot.json\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8uVMKswgmtj6"
      },
      "outputs": [],
      "source": [
        "!DAILY_API_KEY=$DAILY_API_KEY \\\n",
        "  python -m src.cmd.bots.main -f /content/daily_text_glm_voice_bot.json"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hzWlu8YEZZjn"
      },
      "source": [
        "# GLM-4-Voice"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8Ig8oiScyXJH"
      },
      "outputs": [],
      "source": [
        "!cd /content && rm -rf /content/GLM-4-Voice && git clone --recurse-submodules https://github.com/weedge/GLM-4-Voice.git\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iDZvTfQLyeSd"
      },
      "outputs": [],
      "source": [
        "%cd /content/GLM-4-Voice\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fWOBXVkPvfXP"
      },
      "outputs": [],
      "source": [
        "from google.colab import userdata\n",
        "NGROK_TOKEN=userdata.get('NGROK_TOKEN')\n",
        "HF_TOKEN=userdata.get('HF_TOKEN')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yQnRv5HKxULb"
      },
      "outputs": [],
      "source": [
        "!pip install -q gradio pyngrok"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "72nni6ugxTLB"
      },
      "outputs": [],
      "source": [
        "!ngrok config add-authtoken $NGROK_TOKEN"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "U402NpB9vuO9"
      },
      "outputs": [],
      "source": [
        "!cd /content/GLM-4-Voice && python model_server.py --host 0.0.0.0 --model-path /content/achatbot/models/THUDM/glm-4-voice-9b --port 10000 --dtype int4 --device cuda:0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P0i9pHCGwvkr"
      },
      "outputs": [],
      "source": [
        "# open ngrok proxy url to run web gradio ui\n",
        "!cd /content/GLM-4-Voice && python web_demo.py --port 8000 \\\n",
        "  --tokenizer-path  /content/achatbot/models/THUDM/glm-4-voice-tokenizer \\\n",
        "  --model-path /content/achatbot/models/THUDM/glm-4-voice-9b \\\n",
        "  --flow-path /content/achatbot/models/THUDM/glm-4-voice-decoder --ngrok"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "n-ftRxEDGGnT"
      },
      "outputs": [],
      "source": [
        "# open ngrok proxy url to run web gradio ui\n",
        "!cd /content/GLM-4-Voice && nohup python web_demo.py --tokenizer-path  /content/achatbot/models/THUDM/glm-4-voice-tokenizer \\\n",
        "  --model-path /content/achatbot/models/THUDM/glm-4-voice-9b \\\n",
        "  --flow-path /content/achatbot/models/THUDM/glm-4-voice-decoder --ngrok >nohup.out 2>&1 &"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "37YoCH8u-oTr"
      },
      "outputs": [],
      "source": [
        "!cat nohup.out"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v8wt9U2DGlEn"
      },
      "source": [
        "https://dashboard.ngrok.com/agents to stop net agent"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 优化：\n",
        "- glm4-voice 生成的语音速度相对较慢，和单独使用CosyVoice出不多，取决于硬件来加速，或者工程上讲生成的文本和音频分开，并行生成（如果音频依赖问文本，则使用并发）\n",
        "- 生成的音频开始有段卡顿音，需要去除\n",
        "- glm4-voice 模型不支持语音中断, 不过工程上可以实现。"
      ],
      "metadata": {
        "id": "h65sx2cj_A3H"
      }
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "machine_shape": "hm",
      "provenance": [],
      "authorship_tag": "ABX9TyOKZcg06XOtEMrJD2MlPPFj",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}