SDAE_Code_v3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "from tensorflow.keras import regularizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "XL=pd.read_csv('9_510_L_resample_95_5.csv')\n",
    "XU_train=pd.read_csv('9_510_U_resample_95_5.csv')# No resampling done in the unlabeled set, It just denotes the pair of L and U\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle\n",
    "XU_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "XL.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from imblearn.under_sampling import RandomUnderSampler "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a=XL.groupby('ProtocolName')['ProtocolName'].count().sort_values(ascending=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a   # see resampled instances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Function mask_noise\n",
    "\n",
    "Input: numpy array , fraction_delete(between 0 and 1)\n",
    "\n",
    "Removes a fraction_delete fraction of the input array X randomly\n",
    "\n",
    "Output: numpy array after deletions\n",
    "\n",
    "'''\n",
    "def mask_noise(X,fraction_delete=0.2):\n",
    "    mask=np.random.rand(np.shape(X)[0],np.shape(X)[1])\n",
    "    mask=1*(mask>fraction_delete)\n",
    "    return mask*X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Function Gaussian_noise\n",
    "\n",
    "Input: numpy array , sigma\n",
    "\n",
    "adds noise distributed normally with a stddev sigma \n",
    "\n",
    "Output: numpy array after adding noise\n",
    "\n",
    "'''\n",
    "def Gaussian_noise(X,sigma=0.5):\n",
    "    noise = np.random.normal(0,sigma,np.shape(X))\n",
    "    return noise+X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Input, Dense\n",
    "from keras.models import Model\n",
    "import keras.backend as K\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import regularizers\n",
    "from keras.models import load_model\n",
    "import keras\n",
    "from keras.layers import Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Sparcity Autoencoder's Sparcity Activity Regularization\n",
    "\n",
    "'''\n",
    "def kl_divergence(rho, rho_hat):\n",
    "    return rho * tf.math.log(rho) - rho * tf.math.log(rho_hat+1e-12) + (1 - rho) * tf.math.log(1 - rho) - (1 - rho) * tf.math.log(1 - rho_hat+1e-12)\n",
    "\n",
    "class SparseActivityRegularizer(tf.keras.regularizers.Regularizer):\n",
    "\n",
    "    def __init__(self, p=0.1, sparsityBeta=3):\n",
    "        self.p = p\n",
    "        self.sparsityBeta = sparsityBeta\n",
    "\n",
    "    def __call__(self, x):\n",
    "        regularization = 0            \n",
    "\n",
    "        p_hat = K.mean(x, axis=0)\n",
    "        regularization += self.sparsityBeta * K.sum(kl_divergence(self.p, p_hat))\n",
    "\n",
    "        return regularization\n",
    "\n",
    "    def get_config(self):\n",
    "        return {\"name\": self.__class__.__name__} "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.optimizers import RMSprop\n",
    "from keras.optimizers import Adam\n",
    "from keras.optimizers import Adagrad\n",
    "from keras.optimizers import SGD\n",
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yL_train=XL.iloc[:,-1]\n",
    "XL_train=XL.iloc[:,:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yL_train=pd.get_dummies(yL_train)#One-hot_encoding\n",
    "\n",
    "# Split into train,test and val\n",
    "\n",
    "#randomly split into train(80%) and val(20%)\n",
    "XL_train,XL_val,yL_train,yL_val=train_test_split(XL_train,yL_train,random_state=0,test_size=0.2)\n",
    "\n",
    "#randomly split val into val(10%) and test(10%)\n",
    "XL_val,XL_test,yL_val,yL_test=train_test_split(XL_val,yL_val,random_state=0,test_size=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "XL_test.to_csv('XL_test_95L-5U.csv',index=False)\n",
    "yL_test.to_csv('yL_test_95L-5U.csv',index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yL_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.read_csv('yL_test_95L-5U') \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " Reversing the one hot encoding for each set to be used for ML methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yL_train2=yL_train[yL_train==1].stack().reset_index().drop(0,1)['level_1'] # reversing the one hot encoding for each set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yL_val2=yL_val[yL_val==1].stack().reset_index().drop(0,1)['level_1']# reversing the one hot encoding for each set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yL_test2=yL_test[yL_test==1].stack().reset_index().drop(0,1)['level_1']# reversing the one hot encoding for each set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import normalize\n",
    "\n",
    "#scaling and normalization\n",
    "scaler=StandardScaler()\n",
    "XU_train=scaler.fit_transform(XU_train)\n",
    "XL_train=scaler.fit_transform(XL_train)\n",
    "XL_val=scaler.transform(XL_val)\n",
    "XL_test=scaler.transform(XL_test)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Training the SDAE\n",
    "#### Training the Encoder parts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras_sequential_ascii import keras2ascii\n",
    "from tensorflow.keras import regularizers\n",
    "from keras.models import Sequential"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras import backend as K\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Function SDAE\n",
    "Inputs: numpy array X, list layers,list activations,float rho,int n_epochs,string denoise,float fraction_delete\n",
    "\n",
    "denoise is the mode of adding noise\n",
    "rho is the sparcity constraint parameter\n",
    "layers is a list of no. of hidden nodes in each hidden layer\n",
    "activations is a list of activations used in each encoding layer\n",
    "\n",
    "trains each autoencoder layer and outputs all the encoder-layer models.\n",
    "\n",
    "Output: a list of keras models\n",
    "\n",
    "'''\n",
    "def SDAE(X,layers,activations,rho,n_epochs,denoise,fraction_delete=0.2):\n",
    "    layers.insert(0,len(X[0]))\n",
    "    #layers.insert(0,len(X_train.iloc[0]))\n",
    "    ec=[]\n",
    "    for i in range(len(layers)-1):\n",
    "        encoding_dim = layers[i+1]\n",
    "\n",
    "    # this is our input placeholder\n",
    "        input_ = Input(shape=(layers[i],))\n",
    "    # define regulariser\n",
    "        if rho[i] != None: \n",
    "            regulariser = SparseActivityRegularizer(rho[i],3)\n",
    "        else:\n",
    "            regulariser=None\n",
    "\n",
    "        # \"encoded\" is the encoded representation of the input\n",
    "        encoded = Dense(encoding_dim, activation=activations[i],bias_initializer='random_normal',kernel_initializer='he_uniform',activity_regularizer=regulariser,name='encoder'+str(i+1))(input_)\n",
    "        \n",
    "        # \"decoded\" is the lossy reconstruction of the input\n",
    "        decoded = Dense(layers[i],activation=None,name='decoder'+str(i+1))(encoded)\n",
    "\n",
    "        # this model maps an input to its reconstruction\n",
    "        autoencoder = Model(input_, decoded)\n",
    "\n",
    "        # this model maps an input to its encoded representation\n",
    "        encoder = Model(input_, encoded)\n",
    "\n",
    "        # create a placeholder for an encoded (32-dimensional) input\n",
    "        encoded_input = Input(shape=(encoding_dim,))\n",
    "\n",
    "        autoencoder.compile(optimizer=RMSprop(learning_rate=0.001,clipnorm=0.1), loss='mse')\n",
    "        \n",
    "        if denoise =='Gauss':\n",
    "            X_noisy=Gaussian_noise(X,fraction_delete)\n",
    "        else:\n",
    "            X_noisy=mask_noise(X,fraction_delete)\n",
    "        \n",
    "        print('Training Layer ' + str(i+1)+' : ')\n",
    "        autoencoder.fit(X_noisy,X,\n",
    "                epochs=n_epochs,\n",
    "                batch_size=256,\n",
    "                shuffle=True,\n",
    "                validation_data=(X,X))\n",
    "        \n",
    "        ec.append(encoder)\n",
    "        X=encoder.predict(X)\n",
    "    print('All Layers Done!')    \n",
    "    return ec       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Train Sparse Stacked Denoising Autoencoder\n",
    "layers = [100,200,400,50]\n",
    "activations=['relu','relu','relu','relu']\n",
    "rho=[0.06,0.06,0.06,0.06]\n",
    "traint0=time.time()\n",
    "stacked_layers=SDAE(XU_train,layers,activations,rho,10,'mask',0.02)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.Training the SoftMax classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Performs a Forward Pass through the SDAE\n",
    "\n",
    "'''\n",
    "def SDAE_predict(stacked_layers,X):\n",
    "    for layer in stacked_layers:\n",
    "        X1=layer.predict(X)\n",
    "        X=X1\n",
    "    return X1    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#forward pass through SDAE\n",
    "XL_train1=SDAE_predict(stacked_layers,XL_train)\n",
    "XL_val1=SDAE_predict(stacked_layers,XL_val)\n",
    "XL_test1=SDAE_predict(stacked_layers,XL_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Please skip this section to part 6 If only ML methods are to be tested, The following is the DL NN model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Train Classifier separately\n",
    "input1 = Input(shape=(layers[-1],))\n",
    "L1 = Dense(len(yL_train.columns),activation='softmax')(input1)\n",
    "classifier=Model(input1,L1)   \n",
    "classifier.compile(optimizer=SGD(learning_rate=0.1,momentum=0.9),loss='categorical_crossentropy',metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "history=classifier.fit(XL_train1,yL_train,\n",
    "                epochs=50,\n",
    "                #batch_size=256, #default batch_size=32\n",
    "                shuffle=True,\n",
    "                validation_data=(XL_val1, yL_val))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Fine Tuning\n",
    "#### Fine Tuning Involves removing the sparsity regularizer from all the layers , adding dropout etc and performing backpropagation to improve the overall model accuracy "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change regularisation settings\n",
    "a=[]\n",
    "for i in range(len(stacked_layers)):\n",
    "    encoder=stacked_layers[i]\n",
    "    #encoder.layers[-2].activity_regularizer=None  \n",
    "    encoder.layers[-1].activity_regularizer=None #remove Sparse activity regularizer\n",
    "    encoder.save('temp.h5')\n",
    "    encoder=load_model('temp.h5')\n",
    "    print(encoder.losses)\n",
    "    a.append(encoder)\n",
    "    \n",
    "classifier.save('temp.h5')\n",
    "classifier=load_model('temp.h5')\n",
    "print(classifier.losses)\n",
    "a.append(classifier)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add dropout till n-2 layers\n",
    "model=Sequential()\n",
    "for i in a:\n",
    "    model.add(i)\n",
    "    if i != classifier and i != encoder:\n",
    "        model.add(Dropout(0.02))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('temp.h5')\n",
    "model=load_model('temp.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer=RMSprop(learning_rate=0.0001),loss='categorical_crossentropy',metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fine tune model runs for no. of epochs set below\n",
    "history=model.fit(XL_train,yL_train,\n",
    "                epochs=750,\n",
    "                batch_size=256,\n",
    "                shuffle=True,\n",
    "                validation_data=(XL_val, yL_val),\n",
    "                verbose=0)\n",
    "traint1=time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Train_time: \"+str(traint1-traint0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test0=time.time()\n",
    "print(model.evaluate(XL_test,yL_test))\n",
    "test1=time.time()\n",
    "print(\"Test_time: \"+str(test1-test0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Visualise convergence with no. of epochs\n",
    "plt.plot(history.history['accuracy'])\n",
    "plt.plot(history.history['val_accuracy'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Results\n",
    "#### Below are the results obtained on our NN model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred=model.predict(XL_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred1=y_pred.argmax(axis=1)\n",
    "y_pred1=np.append(y_pred1,np.arange(24))\n",
    "y_pred1=pd.get_dummies(y_pred1,columns=yL_train.columns)\n",
    "y_pred1.columns=yL_test.columns\n",
    "y_pred1=y_pred1.iloc[:-24]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yL_test.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test1 = yL_test.columns[np.where(yL_test!=0)[1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred1=y_pred1.columns[np.where(y_pred1!=0)[1]]\n",
    "y_pred1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm=confusion_matrix(y_test1,y_pred1,labels=yL_test.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm_pc=pd.DataFrame((cm.T/np.sum(cm,axis=1)).T,columns=yL_test.columns,index=yL_test.columns)\n",
    "cm_pc=cm_pc.replace(np.nan,0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot confusion matrix\n",
    "plt.figure(figsize=(35,25))\n",
    "df_cm=pd.DataFrame(cm,columns=yL_test.columns,index=yL_test.columns)\n",
    "sns.set(font_scale=1.0)\n",
    "sns.heatmap(cm_pc, annot=True,fmt='.1%',cmap=sns.color_palette(\"Reds\",50)) # font size\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Test Accuracy : '+str(accuracy_score(y_test1, y_pred1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('weighted F1 Score : ' + str(f1_score(y_pred1,y_test1,average = 'weighted')))\n",
    "print('Macro F1 Score : ' + str(f1_score(y_pred1,y_test1,average = 'macro')))\n",
    "print('Micro F1 Score : ' + str(f1_score(y_pred1,y_test1,average = 'micro')))\n",
    "print('F1 Scores : ' )\n",
    "pd.DataFrame(f1_score(y_pred1,y_test1,labels=yL_test.columns,average = None),columns=['F1-score'],index=yL_test.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(yL_train.columns)# No. of distinct labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Weighted Precision : '+str(precision_score(y_test1, y_pred1,average=\"weighted\")))\n",
    "print('Macro Precision : '+str(precision_score(y_test1, y_pred1,average=\"macro\")))\n",
    "print('Micro Precision : '+str(precision_score(y_test1, y_pred1,average=\"micro\")))\n",
    "print('Precision Scores : ' )\n",
    "pd.DataFrame(precision_score(y_pred1,y_test1,labels=yL_test.columns,average = None),columns=['Precision-score'],index=yL_test.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Weighted Recall : '+str(recall_score(y_test1, y_pred1,average=\"weighted\")))\n",
    "print('Macro Recall : '+str(recall_score(y_test1, y_pred1,average=\"macro\")))\n",
    "print('Micro Recall : '+str(recall_score(y_test1, y_pred1,average=\"micro\")))\n",
    "print('Recall Scores : ' )\n",
    "pd.DataFrame(recall_score(y_pred1,y_test1,labels=yL_test.columns,average = None),columns=['recall-score'],index=yL_test.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Confusion Matrix with 10 chosen well-known classes\n",
    "Classes : AMAZON, FACEBOOK, GMAIL, GOOGLE, HTTP, OFFICE_365, SKYPE, TWITTER, WHATSAPP, YOUTUBE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "apps=['AMAZON','FACEBOOK','GMAIL','GOOGLE','HTTP','OFFICE_365','SKYPE','TWITTER','WHATSAPP','YOUTUBE']#reporting for these well-known Apps\n",
    "y_test1[y_test1.isin(apps)]\n",
    "cm=confusion_matrix(y_test1[y_test1.isin(apps)],y_pred1[y_test1.isin(apps)],labels=y_test1[y_test1.isin(apps)].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm_pc=pd.DataFrame((cm.T/np.sum(cm,axis=1)).T,columns=y_test1[y_test1.isin(apps)].unique(),index=y_test1[y_test1.isin(apps)].unique())\n",
    "cm_pc=cm_pc.replace(np.nan,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(15,10))\n",
    "df_cm=pd.DataFrame(cm,columns=y_test1[y_test1.isin(apps)].unique(),index=y_test1[y_test1.isin(apps)].unique())\n",
    "sns.set(font_scale=1.1)\n",
    "sns.heatmap(cm_pc, annot=True,fmt='.1%',cmap=sns.color_palette(\"Reds\",150)) # font size\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. ML methods\n",
    "#### Performing ML methods on Transformed data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import sklearn libraries\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RandomForest\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "clf=RandomForestClassifier(random_state=0, max_depth = 60)\n",
    "clf.fit(XL_train1,yL_train2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(clf.score(XL_train1,yL_train2))# train accuracy\n",
    "print(clf.score(XL_test1,yL_test2))# test accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred=clf.predict(XL_test1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Results\n",
    "print('weighted F1 Score : ' + str(f1_score(y_pred,yL_test2,average='weighted')))\n",
    "print('weighted Precision : ' + str(precision_score(y_pred,yL_test2,average='weighted')))\n",
    "print('weighted Recall : ' + str(recall_score(y_pred,yL_test2,average='weighted')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Results\n",
    "print('macro F1 Score : ' + str(f1_score(y_pred,yL_test2,average='macro')))\n",
    "print('macro Precision : ' + str(precision_score(y_pred,yL_test2,average='macro')))\n",
    "print('macro Recall : ' + str(recall_score(y_pred,yL_test2,average='macro')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "clf=SVC(random_state=0,kernel='rbf').fit(XL_train1,yL_train2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(clf.score(XL_train1,yL_train2))# train accuracy\n",
    "print(clf.score(XL_test1,yL_test2))# test accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred=clf.predict(XL_test1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#results\n",
    "print('weighted F1 Score : ' + str(f1_score(y_pred,yL_test2,average = 'weighted')))\n",
    "print('weighted Precision : ' + str(precision_score(y_pred,yL_test2,average = 'weighted')))\n",
    "print('weighted Recall : ' + str(recall_score(y_pred,yL_test2,average = 'weighted')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#results\n",
    "print('macro F1 Score : ' + str(f1_score(y_pred,yL_test2,average = 'macro')))\n",
    "print('macro' Precision :  + str(precision_score(y_pred,yL_test2,average = 'macro')))\n",
    "print('macro Recall : ' + str(recall_score(y_pred,yL_test2,average = 'macro')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "clf=DecisionTreeClassifier(max_depth=60).fit(XL_train1,yL_train2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.score(XL_train1,yL_train2) #train accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(clf.score(XL_test1,yL_test2))# test accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#results\n",
    "y_pred=clf.predict(XL_test1)\n",
    "print('weighted F1 Score : ' + str(f1_score(y_pred,yL_test2,average = 'weighted')))\n",
    "print('weighted Precision : ' + str(precision_score(y_pred,yL_test2,average = 'weighted')))\n",
    "print('weighted Recall : ' + str(recall_score(y_pred,yL_test2,average = 'weighted')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#results\n",
    "y_pred=clf.predict(XL_test1)\n",
    "print('macro F1 Score : ' + str(f1_score(y_pred,yL_test2,average = 'macro')))\n",
    "print('macro Precision : ' + str(precision_score(y_pred,yL_test2,average = 'macro')))\n",
    "print('macro Recall : ' + str(recall_score(y_pred,yL_test2,average = 'macro')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGBOOST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "from sklearn.multiclass import OneVsRestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train0=time.time()\n",
    "clf = OneVsRestClassifier(XGBClassifier(verbosity=0)).fit(XL_train1, yL_train2)\n",
    "train1=time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Train-score: '+str(clf.score(XL_train1,yL_train2)))# train accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Test-score: '+str(clf.score(XL_test1,yL_test2)))# test accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#results\n",
    "y_pred=clf.predict(XL_test1)\n",
    "print('weighted F1 Score : ' + str(f1_score(y_pred,yL_test2,average = 'weighted')))\n",
    "print('weighted Precision : ' + str(precision_score(y_pred,yL_test2,average = 'weighted')))\n",
    "print('weighted Recall : ' + str(recall_score(y_pred,yL_test2,average = 'weighted')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#results\n",
    "y_pred=clf.predict(XL_test1)\n",
    "print('macro F1 Score : ' + str(f1_score(y_pred,yL_test2,average = 'macro')))\n",
    "print('macro Precision : ' + str(precision_score(y_pred,yL_test2,average = 'maco')))\n",
    "print('macro Recall : ' + str(recall_score(y_pred,yL_test2,average = 'macro')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "clf=DecisionTreeClassifier(max_depth=60).fit(XL_train1,yL_train)\n",
    "print(clf.score(XL_train1,yL_train)) #train accuracy\n",
    "ypred=clf.predict(XL_test1)\n",
    "print(accuracy_score(ypred,yL_test))# test accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(XL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}