Segmentation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Segmentation.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyPWaeh7C07t5l8Pj1YCNYiB",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/muskang48/Speaker-Diarization/blob/master/Segmentation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wNxvXMqOGbs2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%tensorflow_version 1.x\n",
        "import librosa\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from keras.layers.core import Dense\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Bidirectional, TimeDistributed, Dropout\n",
        "from keras.layers import LSTM\n",
        "import numpy as np\n",
        "import keras\n",
        "\n",
        "model = Sequential()\n",
        "\n",
        "model.add(Bidirectional(LSTM(128, return_sequences=True)))\n",
        "model.add(Bidirectional(LSTM(128, return_sequences=True)))\n",
        "model.add(TimeDistributed(Dense(32)))\n",
        "model.add(TimeDistributed(Dense(32)))\n",
        "model.add(TimeDistributed(Dense(1, activation='sigmoid')))\n",
        "\n",
        "model.build(input_shape=(None, 137, 35))\n",
        "model.summary()\n",
        "\n",
        "h5_model_file = '/content/drive/My Drive/SRU/model_hindi_2.h5'\n",
        "model.load_weights(h5_model_file)\n",
        "\n",
        "\n",
        "def multi_segmentation(file):\n",
        "    #sr = 16000\n",
        "    frame_size = 2048\n",
        "    frame_shift = 512\n",
        "    y, sr = librosa.load(file)\n",
        "    mfccs = librosa.feature.mfcc(y, sr, n_mfcc=12, hop_length=frame_shift, n_fft=frame_size)\n",
        "    mfcc_delta = librosa.feature.delta(mfccs)\n",
        "    mfcc_delta2 = librosa.feature.delta(mfccs, order=2)\n",
        "\n",
        "    mfcc = mfccs[1:, ]\n",
        "    norm_mfcc = (mfcc - np.mean(mfcc, axis=1, keepdims=True)) / np.std(mfcc, axis=1, keepdims=True)\n",
        "    norm_mfcc_delta = (mfcc_delta - np.mean(mfcc_delta, axis=1, keepdims=True)) / np.std(mfcc_delta, axis=1, keepdims=True)\n",
        "    norm_mfcc_delta2 = (mfcc_delta2 - np.mean(mfcc_delta2, axis=1, keepdims=True)) / np.std(mfcc_delta2, axis=1, keepdims=True)\n",
        "\n",
        "    ac_feature = np.vstack((norm_mfcc, norm_mfcc_delta, norm_mfcc_delta2))\n",
        "    print(ac_feature.shape)\n",
        "\n",
        "    sub_seq_len = int(3.2 * sr / frame_shift)\n",
        "    sub_seq_step = int(0.8 * sr / frame_shift)\n",
        "\n",
        "    def extract_feature():\n",
        "        feature_len = ac_feature.shape[1]\n",
        "        sub_train_x = []\n",
        "        for i in range(0, feature_len-sub_seq_len, sub_seq_step):\n",
        "            sub_seq_x = np.transpose(ac_feature[:, i: i+sub_seq_len])\n",
        "            sub_train_x.append(sub_seq_x[np.newaxis, :, :])\n",
        "        return np.vstack(sub_train_x), feature_len\n",
        "\n",
        "    predict_x, feature_len = extract_feature()\n",
        "    print(predict_x.shape)\n",
        "\n",
        "    predict_y = model.predict(predict_x)\n",
        "    print(predict_y.shape)\n",
        "\n",
        "    score_acc = np.zeros((feature_len, 1))\n",
        "    score_cnt = np.ones((feature_len, 1))\n",
        "\n",
        "    for i in range(predict_y.shape[0]):\n",
        "        for j in range(predict_y.shape[1]):\n",
        "            index = i*sub_seq_step+j\n",
        "            score_acc[index] += predict_y[i, j, 0]\n",
        "            score_cnt[index] += 1\n",
        "\n",
        "    score_norm = score_acc / score_cnt\n",
        "\n",
        "    wStart = 0\n",
        "    wEnd = 200\n",
        "    wGrow = 200\n",
        "    delta = 25\n",
        "\n",
        "    store_cp = []\n",
        "    index = 0\n",
        "    while wEnd < feature_len:\n",
        "        score_seg = score_norm[wStart:wEnd]\n",
        "        max_v = np.max(score_seg)\n",
        "        max_index = np.argmax(score_seg)\n",
        "        index = index + 1\n",
        "        if max_v > 0.5:\n",
        "            temp = wStart + max_index\n",
        "            store_cp.append(temp)\n",
        "            wStart = wStart + max_index + 50\n",
        "            wEnd = wStart + wGrow\n",
        "        else:\n",
        "            wEnd = wEnd + wGrow\n",
        "\n",
        "    seg_point = np.array(store_cp)*frame_shift\n",
        "\n",
        "    plt.figure('speech segmentation plot')\n",
        "    plt.plot(np.arange(0, len(y)) / (float)(sr), y, \"b-\")\n",
        "\n",
        "    for i in range(len(seg_point)):\n",
        "        plt.vlines(seg_point[i] / (float)(sr), -1, 1, colors=\"c\", linestyles=\"dashed\")\n",
        "        plt.vlines(seg_point[i] / (float)(sr), -1, 1, colors=\"r\", linestyles=\"dashed\")\n",
        "    plt.xlabel(\"Time/s\")\n",
        "    plt.ylabel(\"Speech Amp\")\n",
        "    plt.grid(True)\n",
        "    plt.show()\n",
        "\n",
        "    return np.asarray(seg_point) / float(sr)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}