Abstract_style_SVM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 抽象图像风格分类（SVM版本）\n",
    "## 1.载入数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn import svm, model_selection, metrics\n",
    "from sklearn.model_selection import StratifiedKFold, ParameterGrid\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data_dir = \"/media/ziven/My Passport/Dataset/Painter by Numbers/140x140_style/\"\n",
    "\n",
    "\n",
    "def load_data_PbN(PbN_dir):\n",
    "        \"\"\"load Pinter by Numbers dataset\"\"\"\n",
    "        train_images_Abstract_Art = np.load(os.path.join(PbN_dir, 'style_train_images_Abstract_Art.npy'))\n",
    "        train_images_Abstract_Expressionism = np.load(os.path.join(PbN_dir, 'style_train_images_Abstract_Expressionism.npy'))\n",
    "        train_images_Art_Informel = np.load(os.path.join(PbN_dir, 'style_train_images_Art_Informel.npy'))\n",
    "        train_images_Color_Field_Painting = np.load(os.path.join(PbN_dir, 'style_train_images_Color_Field_Painting.npy'))\n",
    "        train_images_Lyrical_Abstraction = np.load(os.path.join(PbN_dir, 'style_train_images_Lyrical_Abstraction.npy'))\n",
    "        train_images_Concretism = np.load(os.path.join(PbN_dir, 'style_train_images_Concretism.npy'))\n",
    "        train_data = np.concatenate((train_images_Abstract_Art, train_images_Abstract_Expressionism,\n",
    "                                     train_images_Art_Informel, train_images_Color_Field_Painting,\n",
    "                                     train_images_Lyrical_Abstraction, train_images_Concretism))\n",
    "\n",
    "        train_labels_Abstract_Art = [0 for i in range(train_images_Abstract_Art.shape[0])]\n",
    "        train_labels_Abstract_Expressionism = [1 for j in range(train_images_Abstract_Expressionism.shape[0])]\n",
    "        train_labels_Art_Informel = [2 for k in range(train_images_Art_Informel.shape[0])]\n",
    "        train_labels_Color_Field_Painting = [3 for l in range(train_images_Color_Field_Painting.shape[0])]\n",
    "        train_labels_Lyrical_Abstraction = [4 for i1 in range(train_images_Lyrical_Abstraction.shape[0])]\n",
    "        train_labels_Concretism = [5 for i2 in range(train_images_Concretism.shape[0])]\n",
    "        train_labels = np.concatenate((train_labels_Abstract_Art, train_labels_Abstract_Expressionism,\n",
    "                          train_labels_Art_Informel, train_labels_Color_Field_Painting,\n",
    "                          train_labels_Lyrical_Abstraction, train_labels_Concretism))\n",
    "\n",
    "        test_images_Abstract_Art = np.load(os.path.join(PbN_dir, 'style_test_images_Abstract_Art.npy'))\n",
    "        test_images_Abstract_Expressionism = np.load(os.path.join(PbN_dir, 'style_test_images_Abstract_Expressionism.npy'))\n",
    "        test_images_Art_Informel = np.load(os.path.join(PbN_dir, 'style_test_images_Art_Informel.npy'))\n",
    "        test_images_Color_Field_Painting = np.load(os.path.join(PbN_dir, 'style_test_images_Color_Field_Painting.npy'))\n",
    "        test_images_Lyrical_Abstraction = np.load(os.path.join(PbN_dir, 'style_test_images_Lyrical_Abstraction.npy'))\n",
    "        test_images_Concretism = np.load(os.path.join(PbN_dir, 'style_test_images_Concretism.npy'))\n",
    "        val_data = np.concatenate((test_images_Abstract_Art, test_images_Abstract_Expressionism,\n",
    "                                   test_images_Art_Informel, test_images_Color_Field_Painting,\n",
    "                                   test_images_Lyrical_Abstraction, test_images_Concretism))\n",
    "\n",
    "        val_labels_Abstract_Art = [0 for m in range(test_images_Abstract_Art.shape[0])]\n",
    "        val_labels_Abstract_Expressionism = [1 for n in range(test_images_Abstract_Expressionism.shape[0])]\n",
    "        val_labels_Art_Informel = [2 for p in range(test_images_Art_Informel.shape[0])]\n",
    "        val_labels_Color_Field_Painting = [3 for q in range(test_images_Color_Field_Painting.shape[0])]\n",
    "        val_labels_Lyrical_Abstraction = [4 for j1 in range(test_images_Lyrical_Abstraction.shape[0])]\n",
    "        val_labels_Concretism = [5 for j2 in range(test_images_Concretism.shape[0])]\n",
    "        val_labels = np.concatenate((val_labels_Abstract_Art, val_labels_Abstract_Expressionism,\n",
    "                              val_labels_Art_Informel, val_labels_Color_Field_Painting,\n",
    "                              val_labels_Lyrical_Abstraction, val_labels_Concretism))\n",
    "\n",
    "        X = np.concatenate((train_data, val_data))\n",
    "        X = X.reshape([X.shape[0], -1])\n",
    "        y = np.concatenate((train_labels, val_labels))\n",
    "        return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X, y = load_data_PbN(data_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.使用SVM进行分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def SVM_cv(X, y, n_components, decision_function_shape='ovr', kernel='rbf', n_splits=5):\n",
    "    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=2)\n",
    "    avg_accuracy = []\n",
    "    avg_precision = []\n",
    "    avg_recall = []\n",
    "    avg_f_score = []\n",
    "    for train_index, val_index in skf.split(X, y):\n",
    "        pca = PCA(n_components=n_components)\n",
    "        X_train = pca.fit_transform(X[train_index])\n",
    "        X_val = pca.transform(X[val_index])\n",
    "        print(\"X_train shape:\", X_train.shape)\n",
    "        print(\"X_val shape:\", X_val.shape)        \n",
    "        clf = svm.SVC(decision_function_shape='ovr', verbose=2, kernel=kernel)\n",
    "        clf.fit(X_train, y[train_index]) \n",
    "        predicts = clf.predict(X_val)\n",
    "        print(predicts)\n",
    "        print(y[val_index])\n",
    "        ac_score = metrics.accuracy_score(y[val_index], predicts)  \n",
    "        cl_report = metrics.classification_report(y[val_index], predicts)\n",
    "        precision = metrics.precision_score(y[val_index], predicts, average=\"macro\")\n",
    "        recall = metrics.recall_score(y[val_index], predicts, average=\"macro\")\n",
    "        f_score = metrics.f1_score(y[val_index], predicts, average=\"macro\")\n",
    "        \n",
    "        avg_accuracy.append(ac_score)\n",
    "        avg_precision.append(precision)\n",
    "        avg_recall.append(recall)\n",
    "        avg_f_score.append(f_score)\n",
    "        \n",
    "        print(ac_score)\n",
    "        print(cl_report)\n",
    "        print(precision)\n",
    "        print(recall)\n",
    "        print(f_score)\n",
    "    avg_acc = np.average(avg_accuracy)\n",
    "    avg_p = np.average(avg_precision)\n",
    "    avg_r = np.average(avg_recall)\n",
    "    avg_f = np.average(avg_f_score)\n",
    "    print(\"avg_acc:\", avg_acc)\n",
    "    print(\"avg_p:\", avg_p)\n",
    "    print(\"avg_r:\", avg_r)\n",
    "    print(\"avg_f:\", avg_f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 参数设置\n",
    "* n_components：PCA降维后的维度\n",
    "* decision_function_shape：多分类SVM的策略\n",
    "* kernel：SVM的核函数\n",
    "* gamma：RBF核的参数\n",
    "* n_splits：交叉验证的折数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2 进行交叉验证"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_components = 0.9\n",
    "decision_function_shape = 'ovr'\n",
    "kernel = 'rbf'\n",
    "n_splits = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('X_train shape:', (4648, 440))\n",
      "('X_val shape:', (1164, 440))\n",
      "[LibSVM][1 1 1 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.328178694158\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      0.01      0.01       184\n",
      "          1       0.31      0.98      0.47       352\n",
      "          2       0.54      0.03      0.06       208\n",
      "          3       0.60      0.14      0.23       181\n",
      "          4       1.00      0.01      0.02       127\n",
      "          5       0.38      0.03      0.05       112\n",
      "\n",
      "avg / total       0.59      0.33      0.20      1164\n",
      "\n",
      "0.638568267452\n",
      "0.199111249151\n",
      "0.141068307144\n",
      "('X_train shape:', (4649, 448))\n",
      "('X_val shape:', (1163, 448))\n",
      "[LibSVM][1 1 1 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.321582115219\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.00      0.00      0.00       184\n",
      "          1       0.31      0.95      0.47       352\n",
      "          2       0.35      0.06      0.11       208\n",
      "          3       0.57      0.13      0.22       181\n",
      "          4       0.00      0.00      0.00       126\n",
      "          5       1.00      0.04      0.07       112\n",
      "\n",
      "avg / total       0.34      0.32      0.20      1163\n",
      "\n",
      "0.371852209352\n",
      "0.196138949678\n",
      "0.142569733646\n",
      "('X_train shape:', (4649, 444))\n",
      "('X_val shape:', (1163, 444))\n",
      "[LibSVM][1 1 1 ..., 1 5 3]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.32932072227\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      0.01      0.02       184\n",
      "          1       0.32      0.97      0.48       352\n",
      "          2       0.35      0.03      0.06       208\n",
      "          3       0.61      0.15      0.24       181\n",
      "          4       1.00      0.01      0.02       126\n",
      "          5       0.33      0.04      0.08       112\n",
      "\n",
      "avg / total       0.55      0.33      0.21      1163\n",
      "\n",
      "0.602069725937\n",
      "0.202504007861\n",
      "0.148886951288\n",
      "('X_train shape:', (4650, 438))\n",
      "('X_val shape:', (1162, 438))\n",
      "[LibSVM][1 1 1 ..., 1 3 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.327882960413\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      0.01      0.02       183\n",
      "          1       0.31      0.97      0.47       352\n",
      "          2       0.67      0.04      0.07       208\n",
      "          3       0.55      0.12      0.19       181\n",
      "          4       0.50      0.02      0.03       126\n",
      "          5       0.44      0.06      0.11       112\n",
      "\n",
      "avg / total       0.55      0.33      0.20      1162\n",
      "\n",
      "0.578273713719\n",
      "0.202089269255\n",
      "0.149871362879\n",
      "('X_train shape:', (4652, 448))\n",
      "('X_val shape:', (1160, 448))\n",
      "[LibSVM][1 1 1 ..., 1 1 3]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.327586206897\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      0.01      0.01       183\n",
      "          1       0.32      0.96      0.48       351\n",
      "          2       0.58      0.09      0.15       208\n",
      "          3       0.43      0.10      0.16       181\n",
      "          4       0.00      0.00      0.00       126\n",
      "          5       0.42      0.05      0.08       111\n",
      "\n",
      "avg / total       0.46      0.33      0.21      1160\n",
      "\n",
      "0.456863627332\n",
      "0.199909744039\n",
      "0.146547612736\n",
      "('avg_acc:', 0.32691013979139205)\n",
      "('avg_p:', 0.52952550875826043)\n",
      "('avg_r:', 0.1999506439970386)\n",
      "('avg_f:', 0.14578879353857177)\n"
     ]
    }
   ],
   "source": [
    "SVM_cv(X, y, n_components, decision_function_shape, kernel, n_splits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_components = 0.95\n",
    "decision_function_shape = 'ovr'\n",
    "kernel = 'rbf'\n",
    "n_splits = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('X_train shape:', (4648, 981))\n",
      "('X_val shape:', (1164, 981))\n",
      "[LibSVM][1 1 1 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.337628865979\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      0.01      0.01       184\n",
      "          1       0.32      0.95      0.47       352\n",
      "          2       0.35      0.04      0.07       208\n",
      "          3       0.57      0.23      0.32       181\n",
      "          4       1.00      0.04      0.08       127\n",
      "          5       0.57      0.04      0.07       112\n",
      "\n",
      "avg / total       0.57      0.34      0.22      1164\n",
      "\n",
      "0.634164496936\n",
      "0.215727276484\n",
      "0.170266972785\n",
      "('X_train shape:', (4649, 988))\n",
      "('X_val shape:', (1163, 988))\n",
      "[LibSVM][1 1 1 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.337059329321\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.00      0.00      0.00       184\n",
      "          1       0.32      0.93      0.47       352\n",
      "          2       0.42      0.08      0.13       208\n",
      "          3       0.54      0.23      0.32       181\n",
      "          4       0.25      0.01      0.02       126\n",
      "          5       0.70      0.06      0.11       112\n",
      "\n",
      "avg / total       0.35      0.34      0.23      1163\n",
      "\n",
      "0.370849973806\n",
      "0.217590024565\n",
      "0.175883743604\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ziven/anaconda2/lib/python2.7/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/home/ziven/anaconda2/lib/python2.7/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/home/ziven/anaconda2/lib/python2.7/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('X_train shape:', (4649, 982))\n",
      "('X_val shape:', (1163, 982))\n",
      "[LibSVM][1 1 1 ..., 1 5 3]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.337059329321\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.50      0.01      0.02       184\n",
      "          1       0.32      0.93      0.48       352\n",
      "          2       0.42      0.05      0.09       208\n",
      "          3       0.48      0.23      0.31       181\n",
      "          4       0.38      0.02      0.04       126\n",
      "          5       0.35      0.05      0.09       112\n",
      "\n",
      "avg / total       0.40      0.34      0.23      1163\n",
      "\n",
      "0.408280015636\n",
      "0.217052260151\n",
      "0.173183818458\n",
      "('X_train shape:', (4650, 978))\n",
      "('X_val shape:', (1162, 978))\n",
      "[LibSVM][1 1 1 ..., 1 3 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.334767641997\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      0.01      0.02       183\n",
      "          1       0.32      0.93      0.47       352\n",
      "          2       0.62      0.05      0.09       208\n",
      "          3       0.57      0.23      0.33       181\n",
      "          4       0.09      0.01      0.01       126\n",
      "          5       0.25      0.04      0.08       112\n",
      "\n",
      "avg / total       0.49      0.33      0.22      1162\n",
      "\n",
      "0.475021214013\n",
      "0.213048089952\n",
      "0.167286032372\n",
      "('X_train shape:', (4652, 992))\n",
      "('X_val shape:', (1160, 992))\n",
      "[LibSVM][1 1 1 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.331896551724\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.17      0.01      0.01       183\n",
      "          1       0.31      0.91      0.47       351\n",
      "          2       0.58      0.09      0.15       208\n",
      "          3       0.42      0.21      0.28       181\n",
      "          4       0.33      0.01      0.02       126\n",
      "          5       0.56      0.08      0.14       111\n",
      "\n",
      "avg / total       0.38      0.33      0.23      1160\n",
      "\n",
      "0.396496141828\n",
      "0.216158031466\n",
      "0.177470442573\n",
      "('avg_acc:', 0.33568234366830435)\n",
      "('avg_p:', 0.45696236844383809)\n",
      "('avg_r:', 0.21591513652360064)\n",
      "('avg_f:', 0.17281820195849398)\n"
     ]
    }
   ],
   "source": [
    "SVM_cv(X, y, n_components, decision_function_shape, kernel, n_splits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_components = 0.99\n",
    "decision_function_shape = 'ovr'\n",
    "kernel = 'rbf'\n",
    "n_splits = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('X_train shape:', (4648, 2423))\n",
      "('X_val shape:', (1164, 2423))\n",
      "[LibSVM][1 2 1 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.346219931271\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.40      0.01      0.02       184\n",
      "          1       0.32      0.93      0.48       352\n",
      "          2       0.33      0.03      0.05       208\n",
      "          3       0.51      0.30      0.38       181\n",
      "          4       0.40      0.02      0.03       127\n",
      "          5       0.71      0.09      0.16       112\n",
      "\n",
      "avg / total       0.41      0.35      0.24      1164\n",
      "\n",
      "0.446811984587\n",
      "0.229625182865\n",
      "0.186765887931\n",
      "('X_train shape:', (4649, 2418))\n",
      "('X_val shape:', (1163, 2418))\n",
      "[LibSVM][1 1 0 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.345657781599\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.50      0.01      0.02       184\n",
      "          1       0.32      0.91      0.48       352\n",
      "          2       0.53      0.04      0.07       208\n",
      "          3       0.47      0.34      0.39       181\n",
      "          4       0.67      0.02      0.03       126\n",
      "          5       0.88      0.06      0.12       112\n",
      "\n",
      "avg / total       0.50      0.35      0.24      1163\n",
      "\n",
      "0.560334356732\n",
      "0.229915570235\n",
      "0.184558883978\n",
      "('X_train shape:', (4649, 2420))\n",
      "('X_val shape:', (1163, 2420))\n",
      "[LibSVM][1 1 1 ..., 1 1 3]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.331900257954\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.45      0.03      0.05       184\n",
      "          1       0.32      0.89      0.47       352\n",
      "          2       0.44      0.02      0.04       208\n",
      "          3       0.41      0.31      0.36       181\n",
      "          4       0.25      0.02      0.03       126\n",
      "          5       0.36      0.04      0.08       112\n",
      "\n",
      "avg / total       0.37      0.33      0.22      1163\n",
      "\n",
      "0.372056299121\n",
      "0.218507037969\n",
      "0.170353637938\n",
      "('X_train shape:', (4650, 2422))\n",
      "('X_val shape:', (1162, 2422))\n",
      "[LibSVM][1 1 1 ..., 1 3 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.339931153184\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      0.01      0.01       183\n",
      "          1       0.32      0.92      0.48       352\n",
      "          2       0.75      0.03      0.06       208\n",
      "          3       0.52      0.30      0.38       181\n",
      "          4       0.20      0.02      0.03       126\n",
      "          5       0.27      0.06      0.10       112\n",
      "\n",
      "avg / total       0.52      0.34      0.23      1162\n",
      "\n",
      "0.510669239067\n",
      "0.222386941096\n",
      "0.175568287511\n",
      "('X_train shape:', (4652, 2427))\n",
      "('X_val shape:', (1160, 2427))\n",
      "[LibSVM][1 1 1 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.337068965517\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.00      0.00      0.00       183\n",
      "          1       0.32      0.90      0.47       351\n",
      "          2       0.46      0.03      0.05       208\n",
      "          3       0.43      0.30      0.36       181\n",
      "          4       0.25      0.01      0.02       126\n",
      "          5       0.81      0.12      0.20       111\n",
      "\n",
      "avg / total       0.35      0.34      0.23      1160\n",
      "\n",
      "0.379437331492\n",
      "0.22634201375\n",
      "0.183865385926\n",
      "('avg_acc:', 0.34015561790515297)\n",
      "('avg_p:', 0.45386184219976916)\n",
      "('avg_r:', 0.22535534918308917)\n",
      "('avg_f:', 0.18022241665692867)\n"
     ]
    }
   ],
   "source": [
    "SVM_cv(X, y, n_components, decision_function_shape, kernel, n_splits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_components = 0.85\n",
    "decision_function_shape = 'ovr'\n",
    "kernel = 'rbf'\n",
    "n_splits = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('X_train shape:', (4648, 210))\n",
      "('X_val shape:', (1164, 210))\n",
      "[LibSVM][1 1 1 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.317010309278\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      0.01      0.01       184\n",
      "          1       0.31      1.00      0.47       352\n",
      "          2       0.71      0.02      0.05       208\n",
      "          3       0.73      0.06      0.11       181\n",
      "          4       0.00      0.00      0.00       127\n",
      "          5       0.25      0.01      0.02       112\n",
      "\n",
      "avg / total       0.52      0.32      0.17      1164\n",
      "\n",
      "0.501054361938\n",
      "0.182722397858\n",
      "0.109711011653\n",
      "('X_train shape:', (4649, 216))\n",
      "('X_val shape:', (1163, 216))\n",
      "[LibSVM][1 1 1 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.318142734308\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.00      0.00      0.00       184\n",
      "          1       0.31      0.98      0.47       352\n",
      "          2       0.39      0.04      0.08       208\n",
      "          3       0.67      0.07      0.12       181\n",
      "          4       0.00      0.00      0.00       126\n",
      "          5       0.67      0.04      0.07       112\n",
      "\n",
      "avg / total       0.33      0.32      0.18      1163\n",
      "\n",
      "0.338962911018\n",
      "0.187565915898\n",
      "0.122724825187\n",
      "('X_train shape:', (4649, 214))\n",
      "('X_val shape:', (1163, 214))\n",
      "[LibSVM][1 1 1 ..., 1 5 3]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.319002579536\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.67      0.01      0.02       184\n",
      "          1       0.31      0.99      0.47       352\n",
      "          2       0.86      0.03      0.06       208\n",
      "          3       0.61      0.06      0.11       181\n",
      "          4       0.00      0.00      0.00       126\n",
      "          5       0.45      0.04      0.08       112\n",
      "\n",
      "avg / total       0.49      0.32      0.18      1163\n",
      "\n",
      "0.483030825113\n",
      "0.188487918569\n",
      "0.123207934424\n",
      "('X_train shape:', (4650, 210))\n",
      "('X_val shape:', (1162, 210))\n",
      "[LibSVM][1 1 1 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.322719449225\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      0.01      0.02       183\n",
      "          1       0.31      0.98      0.47       352\n",
      "          2       0.73      0.04      0.07       208\n",
      "          3       0.67      0.07      0.12       181\n",
      "          4       0.00      0.00      0.00       126\n",
      "          5       0.50      0.06      0.11       112\n",
      "\n",
      "avg / total       0.53      0.32      0.19      1162\n",
      "\n",
      "0.533949612508\n",
      "0.193523898034\n",
      "0.132910643799\n",
      "('X_train shape:', (4652, 215))\n",
      "('X_val shape:', (1160, 215))\n",
      "[LibSVM][1 1 1 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.313793103448\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      0.01      0.01       183\n",
      "          1       0.31      0.98      0.47       351\n",
      "          2       0.50      0.03      0.06       208\n",
      "          3       0.50      0.04      0.08       181\n",
      "          4       0.00      0.00      0.00       126\n",
      "          5       0.44      0.04      0.07       111\n",
      "\n",
      "avg / total       0.46      0.31      0.17      1160\n",
      "\n",
      "0.458597883598\n",
      "0.183235039691\n",
      "0.114921101755\n",
      "('avg_acc:', 0.31813363515912157)\n",
      "('avg_p:', 0.46311911883493745)\n",
      "('avg_r:', 0.18710703401020637)\n",
      "('avg_f:', 0.12069510336347093)\n"
     ]
    }
   ],
   "source": [
    "SVM_cv(X, y, n_components, decision_function_shape, kernel, n_splits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_components = 0.8\n",
    "decision_function_shape = 'ovr'\n",
    "kernel = 'rbf'\n",
    "n_splits = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('X_train shape:', (4648, 103))\n",
      "('X_val shape:', (1164, 103))\n",
      "[LibSVM][1 1 1 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.310137457045\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      0.01      0.01       184\n",
      "          1       0.31      1.00      0.47       352\n",
      "          2       1.00      0.02      0.05       208\n",
      "          3       0.75      0.02      0.03       181\n",
      "          4       0.00      0.00      0.00       127\n",
      "          5       0.00      0.00      0.00       112\n",
      "\n",
      "avg / total       0.55      0.31      0.16      1164\n",
      "\n",
      "0.509303504199\n",
      "0.174341304964\n",
      "0.0930980233839\n",
      "('X_train shape:', (4649, 106))\n",
      "('X_val shape:', (1163, 106))\n",
      "[LibSVM][1 1 1 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.314703353396\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.00      0.00      0.00       184\n",
      "          1       0.31      1.00      0.47       352\n",
      "          2       0.69      0.04      0.08       208\n",
      "          3       0.62      0.03      0.05       181\n",
      "          4       0.00      0.00      0.00       126\n",
      "          5       0.50      0.01      0.02       112\n",
      "\n",
      "avg / total       0.36      0.31      0.17      1163\n",
      "\n",
      "0.354200404858\n",
      "0.179496867083\n",
      "0.103735209956\n",
      "('X_train shape:', (4649, 105))\n",
      "('X_val shape:', (1163, 105))\n",
      "[LibSVM][1 1 1 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.311263972485\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      0.01      0.02       184\n",
      "          1       0.31      0.99      0.47       352\n",
      "          2       1.00      0.02      0.04       208\n",
      "          3       0.71      0.03      0.05       181\n",
      "          4       0.00      0.00      0.00       126\n",
      "          5       0.29      0.02      0.03       112\n",
      "\n",
      "avg / total       0.57      0.31      0.16      1163\n",
      "\n",
      "0.550889472149\n",
      "0.177843176571\n",
      "0.102155965375\n",
      "('X_train shape:', (4650, 103))\n",
      "('X_val shape:', (1162, 103))\n",
      "[LibSVM][1 1 1 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.31669535284\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      0.01      0.02       183\n",
      "          1       0.31      1.00      0.47       352\n",
      "          2       1.00      0.03      0.06       208\n",
      "          3       0.67      0.02      0.04       181\n",
      "          4       0.00      0.00      0.00       126\n",
      "          5       0.67      0.04      0.07       112\n",
      "\n",
      "avg / total       0.60      0.32      0.17      1162\n",
      "\n",
      "0.606927417786\n",
      "0.182931474804\n",
      "0.109915325495\n",
      "('X_train shape:', (4652, 105))\n",
      "('X_val shape:', (1160, 105))\n",
      "[LibSVM][1 1 1 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.312068965517\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      0.01      0.01       183\n",
      "          1       0.31      0.99      0.47       351\n",
      "          2       0.78      0.03      0.06       208\n",
      "          3       0.40      0.01      0.02       181\n",
      "          4       0.00      0.00      0.00       126\n",
      "          5       0.43      0.03      0.05       111\n",
      "\n",
      "avg / total       0.49      0.31      0.16      1160\n",
      "\n",
      "0.485504598246\n",
      "0.178582845352\n",
      "0.10275158591\n",
      "('avg_acc:', 0.3129738202566375)\n",
      "('avg_p:', 0.50136507944773467)\n",
      "('avg_r:', 0.17863913375478305)\n",
      "('avg_f:', 0.10233122202390595)\n"
     ]
    }
   ],
   "source": [
    "SVM_cv(X, y, n_components, decision_function_shape, kernel, n_splits)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 3.使用LR进行分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def LR_cv(X, y, n_components, C=1.0, solver=\"saga\", multi_class=\"multinomial\", n_splits=5):\n",
    "    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=2)\n",
    "    avg_accuracy = []\n",
    "    avg_precision = []\n",
    "    avg_recall = []\n",
    "    avg_f_score = []\n",
    "    for train_index, val_index in skf.split(X, y):\n",
    "        # PCA\n",
    "        pca = PCA(n_components=n_components)\n",
    "        X_train = pca.fit_transform(X[train_index])\n",
    "        X_val = pca.transform(X[val_index])\n",
    "        print(\"X_train shape:\", X_train.shape)\n",
    "        print(\"X_val shape:\", X_val.shape)  \n",
    "        \n",
    "        clf = LogisticRegression(C=C, solver=solver, multi_class=multi_class, n_jobs=-1)\n",
    "        clf.fit(X_train, y[train_index]) \n",
    "        predicts = clf.predict(X_val)\n",
    "        print(predicts)\n",
    "        print(y[val_index])\n",
    "        ac_score = metrics.accuracy_score(y[val_index], predicts)  \n",
    "        cl_report = metrics.classification_report(y[val_index], predicts)\n",
    "        precision = metrics.precision_score(y[val_index], predicts, average=\"macro\")\n",
    "        recall = metrics.recall_score(y[val_index], predicts, average=\"macro\")\n",
    "        f_score = metrics.f1_score(y[val_index], predicts, average=\"macro\")\n",
    "        \n",
    "        avg_accuracy.append(ac_score)\n",
    "        avg_precision.append(precision)\n",
    "        avg_recall.append(recall)\n",
    "        avg_f_score.append(f_score)\n",
    "        \n",
    "        print(ac_score)\n",
    "        print(cl_report)\n",
    "        print(precision)\n",
    "        print(recall)\n",
    "        print(f_score)\n",
    "    avg_acc = np.average(avg_accuracy)\n",
    "    avg_p = np.average(avg_precision)\n",
    "    avg_r = np.average(avg_recall)\n",
    "    avg_f = np.average(avg_f_score)\n",
    "    print(\"avg_acc:\", avg_acc)\n",
    "    print(\"avg_p:\", avg_p)\n",
    "    print(\"avg_r:\", avg_r)\n",
    "    print(\"avg_f:\", avg_f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_components = 0.95"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('X_train shape:', (4648, 981))\n",
      "('X_val shape:', (1164, 981))\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ziven/anaconda2/lib/python2.7/site-packages/sklearn/linear_model/sag.py:326: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 1 0 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.254295532646\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.22      0.21      0.21       184\n",
      "          1       0.29      0.39      0.33       352\n",
      "          2       0.26      0.22      0.24       208\n",
      "          3       0.27      0.27      0.27       181\n",
      "          4       0.14      0.09      0.11       127\n",
      "          5       0.19      0.13      0.16       112\n",
      "\n",
      "avg / total       0.24      0.25      0.25      1164\n",
      "\n",
      "0.228111209534\n",
      "0.218415043552\n",
      "0.220304142928\n",
      "('X_train shape:', (4649, 988))\n",
      "('X_val shape:', (1163, 988))\n",
      "[3 2 0 ..., 4 3 2]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.248495270851\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.23      0.18      0.21       184\n",
      "          1       0.29      0.38      0.33       352\n",
      "          2       0.22      0.21      0.21       208\n",
      "          3       0.26      0.28      0.27       181\n",
      "          4       0.17      0.11      0.13       126\n",
      "          5       0.14      0.12      0.13       112\n",
      "\n",
      "avg / total       0.24      0.25      0.24      1163\n",
      "\n",
      "0.220409343748\n",
      "0.213524281932\n",
      "0.214307002587\n",
      "('X_train shape:', (4649, 982))\n",
      "('X_val shape:', (1163, 982))\n",
      "[3 3 2 ..., 2 1 3]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.250214961307\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.19      0.17      0.18       184\n",
      "          1       0.30      0.42      0.35       352\n",
      "          2       0.25      0.22      0.23       208\n",
      "          3       0.26      0.25      0.25       181\n",
      "          4       0.13      0.10      0.11       126\n",
      "          5       0.15      0.08      0.11       112\n",
      "\n",
      "avg / total       0.24      0.25      0.24      1163\n",
      "\n",
      "0.214569042699\n",
      "0.206149091437\n",
      "0.206089112114\n",
      "('X_train shape:', (4650, 978))\n",
      "('X_val shape:', (1162, 978))\n",
      "[1 3 1 ..., 2 1 2]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.251290877797\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.25      0.19      0.22       183\n",
      "          1       0.30      0.41      0.35       352\n",
      "          2       0.20      0.19      0.20       208\n",
      "          3       0.26      0.26      0.26       181\n",
      "          4       0.15      0.13      0.14       126\n",
      "          5       0.16      0.11      0.13       112\n",
      "\n",
      "avg / total       0.24      0.25      0.24      1162\n",
      "\n",
      "0.221824617511\n",
      "0.213133720503\n",
      "0.214476105098\n",
      "('X_train shape:', (4652, 992))\n",
      "('X_val shape:', (1160, 992))\n",
      "[0 1 0 ..., 0 3 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.260344827586\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.24      0.20      0.21       183\n",
      "          1       0.30      0.40      0.34       351\n",
      "          2       0.21      0.20      0.21       208\n",
      "          3       0.29      0.29      0.29       181\n",
      "          4       0.14      0.10      0.11       126\n",
      "          5       0.24      0.18      0.21       111\n",
      "\n",
      "avg / total       0.25      0.26      0.25      1160\n",
      "\n",
      "0.237092874547\n",
      "0.227148623231\n",
      "0.229428193698\n",
      "('avg_acc:', 0.25292829403747363)\n",
      "('avg_p:', 0.22440141760800678)\n",
      "('avg_r:', 0.21567415213081045)\n",
      "('avg_f:', 0.21692091128497309)\n"
     ]
    }
   ],
   "source": [
    "LR_cv(X, y, n_components)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('X_train shape:', (4648, 2423))\n",
      "('X_val shape:', (1164, 2423))\n",
      "[1 2 1 ..., 3 3 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.233676975945\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.18      0.16      0.17       184\n",
      "          1       0.29      0.32      0.30       352\n",
      "          2       0.22      0.20      0.21       208\n",
      "          3       0.23      0.28      0.25       181\n",
      "          4       0.19      0.16      0.17       127\n",
      "          5       0.19      0.14      0.16       112\n",
      "\n",
      "avg / total       0.23      0.23      0.23      1164\n",
      "\n",
      "0.215042490127\n",
      "0.21091680376\n",
      "0.211397547846\n",
      "('X_train shape:', (4649, 2418))\n",
      "('X_val shape:', (1163, 2418))\n",
      "[3 2 0 ..., 0 3 4]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.249355116079\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.23      0.16      0.19       184\n",
      "          1       0.29      0.35      0.32       352\n",
      "          2       0.21      0.19      0.20       208\n",
      "          3       0.26      0.33      0.29       181\n",
      "          4       0.18      0.13      0.15       126\n",
      "          5       0.21      0.20      0.20       112\n",
      "\n",
      "avg / total       0.24      0.25      0.24      1163\n",
      "\n",
      "0.230120984688\n",
      "0.225708769544\n",
      "0.224840212867\n",
      "('X_train shape:', (4649, 2420))\n",
      "('X_val shape:', (1163, 2420))\n",
      "[0 3 2 ..., 2 1 3]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.241616509028\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.19      0.16      0.18       184\n",
      "          1       0.29      0.37      0.32       352\n",
      "          2       0.20      0.17      0.18       208\n",
      "          3       0.29      0.34      0.31       181\n",
      "          4       0.13      0.10      0.12       126\n",
      "          5       0.16      0.11      0.13       112\n",
      "\n",
      "avg / total       0.23      0.24      0.23      1163\n",
      "\n",
      "0.210621773644\n",
      "0.208441479756\n",
      "0.206893985426\n",
      "('X_train shape:', (4650, 2422))\n",
      "('X_val shape:', (1162, 2422))\n",
      "[1 3 1 ..., 1 1 2]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.238382099828\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.22      0.19      0.20       183\n",
      "          1       0.30      0.36      0.33       352\n",
      "          2       0.19      0.19      0.19       208\n",
      "          3       0.25      0.29      0.27       181\n",
      "          4       0.11      0.09      0.10       126\n",
      "          5       0.19      0.12      0.14       112\n",
      "\n",
      "avg / total       0.23      0.24      0.23      1162\n",
      "\n",
      "0.210021865867\n",
      "0.20537421382\n",
      "0.205127985568\n",
      "('X_train shape:', (4652, 2427))\n",
      "('X_val shape:', (1160, 2427))\n",
      "[1 4 4 ..., 0 3 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.244827586207\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.19      0.15      0.17       183\n",
      "          1       0.32      0.37      0.34       351\n",
      "          2       0.21      0.19      0.20       208\n",
      "          3       0.26      0.31      0.28       181\n",
      "          4       0.12      0.10      0.10       126\n",
      "          5       0.20      0.15      0.17       111\n",
      "\n",
      "avg / total       0.24      0.24      0.24      1160\n",
      "\n",
      "0.214281214258\n",
      "0.213165317104\n",
      "0.211916882396\n",
      "('avg_acc:', 0.24157165741745548)\n",
      "('avg_p:', 0.21601766571690284)\n",
      "('avg_r:', 0.21272131679667669)\n",
      "('avg_f:', 0.21203532282069232)\n"
     ]
    }
   ],
   "source": [
    "LR_cv(X, y, 0.99)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('X_train shape:', (4648, 440))\n",
      "('X_val shape:', (1164, 440))\n",
      "[3 1 0 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.27147766323\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.22      0.20      0.21       184\n",
      "          1       0.31      0.52      0.39       352\n",
      "          2       0.23      0.16      0.19       208\n",
      "          3       0.31      0.23      0.26       181\n",
      "          4       0.14      0.08      0.10       127\n",
      "          5       0.21      0.11      0.14       112\n",
      "\n",
      "avg / total       0.25      0.27      0.25      1164\n",
      "\n",
      "0.235488143414\n",
      "0.215785578457\n",
      "0.215411885423\n",
      "('X_train shape:', (4649, 448))\n",
      "('X_val shape:', (1163, 448))\n",
      "[3 2 3 ..., 0 5 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.263112639725\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.22      0.17      0.19       184\n",
      "          1       0.29      0.46      0.36       352\n",
      "          2       0.24      0.20      0.22       208\n",
      "          3       0.27      0.24      0.25       181\n",
      "          4       0.16      0.10      0.12       126\n",
      "          5       0.25      0.14      0.18       112\n",
      "\n",
      "avg / total       0.25      0.26      0.25      1163\n",
      "\n",
      "0.239903284048\n",
      "0.217715484898\n",
      "0.220713500446\n",
      "('X_train shape:', (4649, 444))\n",
      "('X_val shape:', (1163, 444))\n",
      "[3 3 5 ..., 4 1 3]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.246775580396\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.13      0.10      0.11       184\n",
      "          1       0.31      0.51      0.39       352\n",
      "          2       0.19      0.16      0.17       208\n",
      "          3       0.30      0.23      0.26       181\n",
      "          4       0.07      0.04      0.05       126\n",
      "          5       0.15      0.07      0.10       112\n",
      "\n",
      "avg / total       0.22      0.25      0.22      1163\n",
      "\n",
      "0.190428920658\n",
      "0.185639964762\n",
      "0.179766295616\n",
      "('X_train shape:', (4650, 438))\n",
      "('X_val shape:', (1162, 438))\n",
      "[2 2 1 ..., 1 1 2]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.260757314974\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.23      0.17      0.19       183\n",
      "          1       0.31      0.50      0.38       352\n",
      "          2       0.23      0.21      0.22       208\n",
      "          3       0.27      0.20      0.23       181\n",
      "          4       0.10      0.06      0.07       126\n",
      "          5       0.16      0.08      0.11       112\n",
      "\n",
      "avg / total       0.24      0.26      0.24      1162\n",
      "\n",
      "0.215283748566\n",
      "0.202743710708\n",
      "0.200310941245\n",
      "('X_train shape:', (4652, 448))\n",
      "('X_val shape:', (1160, 448))\n",
      "[0 0 0 ..., 5 2 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.261206896552\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.22      0.20      0.21       183\n",
      "          1       0.29      0.46      0.36       351\n",
      "          2       0.22      0.17      0.19       208\n",
      "          3       0.30      0.23      0.26       181\n",
      "          4       0.12      0.07      0.09       126\n",
      "          5       0.26      0.16      0.20       111\n",
      "\n",
      "avg / total       0.25      0.26      0.25      1160\n",
      "\n",
      "0.235838349938\n",
      "0.215835489853\n",
      "0.218394142002\n",
      "('avg_acc:', 0.26066601897530506)\n",
      "('avg_p:', 0.22338848932477395)\n",
      "('avg_r:', 0.20754404573556157)\n",
      "('avg_f:', 0.20691935294644836)\n"
     ]
    }
   ],
   "source": [
    "LR_cv(X, y, 0.90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('X_train shape:', (4648, 210))\n",
      "('X_val shape:', (1164, 210))\n",
      "[3 1 1 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.294673539519\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.21      0.16      0.18       184\n",
      "          1       0.32      0.64      0.42       352\n",
      "          2       0.30      0.15      0.20       208\n",
      "          3       0.27      0.19      0.22       181\n",
      "          4       0.20      0.07      0.10       127\n",
      "          5       0.34      0.14      0.20       112\n",
      "\n",
      "avg / total       0.28      0.29      0.26      1164\n",
      "\n",
      "0.273252174285\n",
      "0.224096563669\n",
      "0.221777027614\n",
      "('X_train shape:', (4649, 216))\n",
      "('X_val shape:', (1163, 216))\n",
      "[3 0 3 ..., 2 5 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.292347377472\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.24      0.15      0.18       184\n",
      "          1       0.31      0.62      0.41       352\n",
      "          2       0.22      0.15      0.18       208\n",
      "          3       0.32      0.24      0.27       181\n",
      "          4       0.27      0.05      0.08       126\n",
      "          5       0.33      0.12      0.18       112\n",
      "\n",
      "avg / total       0.28      0.29      0.25      1163\n",
      "\n",
      "0.281946115775\n",
      "0.221786444132\n",
      "0.218440382291\n",
      "('X_train shape:', (4649, 214))\n",
      "('X_val shape:', (1163, 214))\n",
      "[3 3 5 ..., 2 1 3]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.273430782459\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.19      0.11      0.14       184\n",
      "          1       0.32      0.64      0.42       352\n",
      "          2       0.18      0.14      0.16       208\n",
      "          3       0.33      0.20      0.25       181\n",
      "          4       0.00      0.00      0.00       126\n",
      "          5       0.16      0.05      0.08       112\n",
      "\n",
      "avg / total       0.23      0.27      0.23      1163\n",
      "\n",
      "0.196628583541\n",
      "0.191359250286\n",
      "0.175449499803\n",
      "('X_train shape:', (4650, 210))\n",
      "('X_val shape:', (1162, 210))\n",
      "[1 0 1 ..., 1 1 2]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.29604130809\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.31      0.19      0.23       183\n",
      "          1       0.31      0.65      0.42       352\n",
      "          2       0.25      0.12      0.17       208\n",
      "          3       0.32      0.22      0.26       181\n",
      "          4       0.13      0.05      0.07       126\n",
      "          5       0.19      0.07      0.10       112\n",
      "\n",
      "avg / total       0.27      0.30      0.25      1162\n",
      "\n",
      "0.252289902309\n",
      "0.21737392247\n",
      "0.209670081979\n",
      "('X_train shape:', (4652, 215))\n",
      "('X_val shape:', (1160, 215))\n",
      "[3 0 0 ..., 4 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.279310344828\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.20      0.16      0.18       183\n",
      "          1       0.31      0.61      0.41       351\n",
      "          2       0.20      0.11      0.14       208\n",
      "          3       0.30      0.20      0.24       181\n",
      "          4       0.14      0.05      0.07       126\n",
      "          5       0.29      0.14      0.19       111\n",
      "\n",
      "avg / total       0.25      0.28      0.24      1160\n",
      "\n",
      "0.241884012479\n",
      "0.210984591729\n",
      "0.205484863144\n",
      "('avg_acc:', 0.28716067047343996)\n",
      "('avg_p:', 0.24920015767784598)\n",
      "('avg_r:', 0.21312015445725546)\n",
      "('avg_f:', 0.20616437096613627)\n"
     ]
    }
   ],
   "source": [
    "LR_cv(X, y, 0.85)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('X_train shape:', (4648, 103))\n",
      "('X_val shape:', (1164, 103))\n",
      "[3 3 1 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.308419243986\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.20      0.11      0.14       184\n",
      "          1       0.32      0.75      0.45       352\n",
      "          2       0.26      0.11      0.15       208\n",
      "          3       0.37      0.20      0.26       181\n",
      "          4       0.29      0.05      0.08       127\n",
      "          5       0.40      0.09      0.15       112\n",
      "\n",
      "avg / total       0.30      0.31      0.25      1164\n",
      "\n",
      "0.30531627642\n",
      "0.218001409123\n",
      "0.205261217328\n",
      "('X_train shape:', (4649, 106))\n",
      "('X_val shape:', (1163, 106))\n",
      "[3 1 0 ..., 1 2 2]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.299226139295\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.20      0.11      0.14       184\n",
      "          1       0.31      0.74      0.44       352\n",
      "          2       0.23      0.10      0.14       208\n",
      "          3       0.30      0.18      0.23       181\n",
      "          4       0.36      0.03      0.06       126\n",
      "          5       0.42      0.09      0.15       112\n",
      "\n",
      "avg / total       0.29      0.30      0.24      1163\n",
      "\n",
      "0.304129853253\n",
      "0.208607623715\n",
      "0.192390517291\n",
      "('X_train shape:', (4649, 105))\n",
      "('X_val shape:', (1163, 105))\n",
      "[1 3 1 ..., 1 1 3]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.298366294067\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.23      0.11      0.15       184\n",
      "          1       0.32      0.75      0.44       352\n",
      "          2       0.20      0.11      0.14       208\n",
      "          3       0.38      0.19      0.26       181\n",
      "          4       0.04      0.01      0.01       126\n",
      "          5       0.30      0.05      0.09       112\n",
      "\n",
      "avg / total       0.26      0.30      0.23      1163\n",
      "\n",
      "0.245560708582\n",
      "0.202750346018\n",
      "0.182025242006\n",
      "('X_train shape:', (4650, 103))\n",
      "('X_val shape:', (1162, 103))\n",
      "[1 3 1 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.298623063683\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.25      0.10      0.15       183\n",
      "          1       0.31      0.74      0.44       352\n",
      "          2       0.30      0.10      0.15       208\n",
      "          3       0.33      0.20      0.25       181\n",
      "          4       0.23      0.04      0.07       126\n",
      "          5       0.13      0.04      0.07       112\n",
      "\n",
      "avg / total       0.28      0.30      0.23      1162\n",
      "\n",
      "0.25804121833\n",
      "0.204914062042\n",
      "0.185962348832\n",
      "('X_train shape:', (4652, 105))\n",
      "('X_val shape:', (1160, 105))\n",
      "[3 1 0 ..., 3 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.288793103448\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.18      0.11      0.14       183\n",
      "          1       0.30      0.69      0.42       351\n",
      "          2       0.22      0.09      0.13       208\n",
      "          3       0.29      0.18      0.22       181\n",
      "          4       0.17      0.03      0.05       126\n",
      "          5       0.53      0.16      0.25       111\n",
      "\n",
      "avg / total       0.27      0.29      0.24      1160\n",
      "\n",
      "0.28231296537\n",
      "0.21057901564\n",
      "0.20215078039\n",
      "('avg_acc:', 0.29868556889596592)\n",
      "('avg_p:', 0.27907220439094027)\n",
      "('avg_r:', 0.20897049130753328)\n",
      "('avg_f:', 0.19355802116926996)\n"
     ]
    }
   ],
   "source": [
    "LR_cv(X, y, 0.80)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.使用RF进行分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def RF_cv(X, y, n_components, n_estimators=10, max_depth=None, min_samples_split=2, n_splits=5):\n",
    "    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=2)\n",
    "    avg_accuracy = []\n",
    "    avg_precision = []\n",
    "    avg_recall = []\n",
    "    avg_f_score = []\n",
    "    for train_index, val_index in skf.split(X, y):\n",
    "        # PCA\n",
    "        pca = PCA(n_components=n_components)\n",
    "        X_train = pca.fit_transform(X[train_index])\n",
    "        X_val = pca.transform(X[val_index])\n",
    "        print(\"X_train shape:\", X_train.shape)\n",
    "        print(\"X_val shape:\", X_val.shape)  \n",
    "        \n",
    "        clf = RandomForestClassifier(n_estimators=n_estimators, max_depth=max_depth, min_samples_split=min_samples_split, random_state=0, n_jobs=-1)\n",
    "        clf.fit(X_train, y[train_index]) \n",
    "        predicts = clf.predict(X_val)\n",
    "        print(predicts)\n",
    "        print(y[val_index])\n",
    "        ac_score = metrics.accuracy_score(y[val_index], predicts)  \n",
    "        cl_report = metrics.classification_report(y[val_index], predicts)\n",
    "        precision = metrics.precision_score(y[val_index], predicts, average=\"macro\")\n",
    "        recall = metrics.recall_score(y[val_index], predicts, average=\"macro\")\n",
    "        f_score = metrics.f1_score(y[val_index], predicts, average=\"macro\")\n",
    "        \n",
    "        avg_accuracy.append(ac_score)\n",
    "        avg_precision.append(precision)\n",
    "        avg_recall.append(recall)\n",
    "        avg_f_score.append(f_score)\n",
    "        \n",
    "        print(ac_score)\n",
    "        print(cl_report)\n",
    "        print(precision)\n",
    "        print(recall)\n",
    "        print(f_score)\n",
    "    avg_acc = np.average(avg_accuracy)\n",
    "    avg_p = np.average(avg_precision)\n",
    "    avg_r = np.average(avg_recall)\n",
    "    avg_f = np.average(avg_f_score)\n",
    "    print(\"avg_acc:\", avg_acc)\n",
    "    print(\"avg_p:\", avg_p)\n",
    "    print(\"avg_r:\", avg_r)\n",
    "    print(\"avg_f:\", avg_f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_components = 0.95"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('X_train shape:', (4648, 981))\n",
      "('X_val shape:', (1164, 981))\n",
      "[2 1 0 ..., 2 0 2]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.274914089347\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.22      0.27      0.24       184\n",
      "          1       0.33      0.52      0.40       352\n",
      "          2       0.21      0.14      0.17       208\n",
      "          3       0.30      0.27      0.28       181\n",
      "          4       0.15      0.07      0.10       127\n",
      "          5       0.05      0.01      0.02       112\n",
      "\n",
      "avg / total       0.24      0.27      0.25      1164\n",
      "\n",
      "0.210435353375\n",
      "0.213000572923\n",
      "0.201612050289\n",
      "('X_train shape:', (4649, 988))\n",
      "('X_val shape:', (1163, 988))\n",
      "[3 1 1 ..., 0 4 0]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.282889079966\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.20      0.22      0.21       184\n",
      "          1       0.32      0.53      0.40       352\n",
      "          2       0.26      0.21      0.23       208\n",
      "          3       0.36      0.27      0.31       181\n",
      "          4       0.10      0.04      0.06       126\n",
      "          5       0.15      0.04      0.07       112\n",
      "\n",
      "avg / total       0.26      0.28      0.26      1163\n",
      "\n",
      "0.231424721792\n",
      "0.218730414277\n",
      "0.212849176676\n",
      "('X_train shape:', (4649, 982))\n",
      "('X_val shape:', (1163, 982))\n",
      "[0 1 0 ..., 2 1 4]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.264832330181\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.21      0.24      0.22       184\n",
      "          1       0.31      0.51      0.39       352\n",
      "          2       0.22      0.18      0.20       208\n",
      "          3       0.30      0.22      0.25       181\n",
      "          4       0.14      0.06      0.08       126\n",
      "          5       0.07      0.02      0.03       112\n",
      "\n",
      "avg / total       0.24      0.26      0.24      1163\n",
      "\n",
      "0.208650814734\n",
      "0.202403348185\n",
      "0.194133327119\n",
      "('X_train shape:', (4650, 978))\n",
      "('X_val shape:', (1162, 978))\n",
      "[1 3 3 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.261617900172\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.18      0.19      0.18       183\n",
      "          1       0.31      0.51      0.38       352\n",
      "          2       0.22      0.16      0.18       208\n",
      "          3       0.31      0.28      0.29       181\n",
      "          4       0.12      0.05      0.07       126\n",
      "          5       0.09      0.03      0.04       112\n",
      "\n",
      "avg / total       0.23      0.26      0.23      1162\n",
      "\n",
      "0.204578140224\n",
      "0.200129311648\n",
      "0.191873536692\n",
      "('X_train shape:', (4652, 992))\n",
      "('X_val shape:', (1160, 992))\n",
      "[3 1 3 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.263793103448\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.19      0.22      0.21       183\n",
      "          1       0.31      0.50      0.39       351\n",
      "          2       0.26      0.19      0.22       208\n",
      "          3       0.28      0.23      0.25       181\n",
      "          4       0.00      0.00      0.00       126\n",
      "          5       0.19      0.05      0.08       111\n",
      "\n",
      "avg / total       0.23      0.26      0.24      1160\n",
      "\n",
      "0.206920510884\n",
      "0.201120527563\n",
      "0.19222512785\n",
      "('avg_acc:', 0.26960930062272914)\n",
      "('avg_p:', 0.21240190820162322)\n",
      "('avg_r:', 0.2070768349194009)\n",
      "('avg_f:', 0.1985386437252453)\n"
     ]
    }
   ],
   "source": [
    "RF_cv(X, y, n_components)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('X_train shape:', (4648, 2423))\n",
      "('X_val shape:', (1164, 2423))\n",
      "[1 0 1 ..., 0 1 0]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.254295532646\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.19      0.24      0.21       184\n",
      "          1       0.31      0.47      0.37       352\n",
      "          2       0.17      0.12      0.14       208\n",
      "          3       0.32      0.31      0.31       181\n",
      "          4       0.07      0.02      0.04       127\n",
      "          5       0.04      0.01      0.01       112\n",
      "\n",
      "avg / total       0.22      0.25      0.23      1164\n",
      "\n",
      "0.18353765273\n",
      "0.19638188634\n",
      "0.181910794555\n",
      "('X_train shape:', (4649, 2418))\n",
      "('X_val shape:', (1163, 2418))\n",
      "[1 1 0 ..., 1 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.263112639725\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.17      0.18      0.18       184\n",
      "          1       0.32      0.48      0.38       352\n",
      "          2       0.20      0.16      0.18       208\n",
      "          3       0.32      0.31      0.31       181\n",
      "          4       0.12      0.05      0.07       126\n",
      "          5       0.18      0.05      0.08       112\n",
      "\n",
      "avg / total       0.24      0.26      0.24      1163\n",
      "\n",
      "0.217671654343\n",
      "0.206963572333\n",
      "0.200844819653\n",
      "('X_train shape:', (4649, 2420))\n",
      "('X_val shape:', (1163, 2420))\n",
      "[1 0 0 ..., 2 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.235597592433\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.19      0.22      0.20       184\n",
      "          1       0.27      0.44      0.34       352\n",
      "          2       0.17      0.12      0.14       208\n",
      "          3       0.31      0.25      0.28       181\n",
      "          4       0.11      0.05      0.07       126\n",
      "          5       0.07      0.02      0.03       112\n",
      "\n",
      "avg / total       0.21      0.24      0.21      1163\n",
      "\n",
      "0.185814901224\n",
      "0.183356371922\n",
      "0.175597723906\n",
      "('X_train shape:', (4650, 2422))\n",
      "('X_val shape:', (1162, 2422))\n",
      "[1 1 1 ..., 0 3 2]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.269363166954\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.17      0.19      0.18       183\n",
      "          1       0.32      0.53      0.40       352\n",
      "          2       0.23      0.16      0.19       208\n",
      "          3       0.32      0.29      0.30       181\n",
      "          4       0.13      0.06      0.08       126\n",
      "          5       0.04      0.01      0.01       112\n",
      "\n",
      "avg / total       0.23      0.27      0.24      1162\n",
      "\n",
      "0.201428614042\n",
      "0.204552697373\n",
      "0.193752961763\n",
      "('X_train shape:', (4652, 2427))\n",
      "('X_val shape:', (1160, 2427))\n",
      "[1 1 1 ..., 0 0 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.256034482759\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.21      0.25      0.23       183\n",
      "          1       0.31      0.48      0.38       351\n",
      "          2       0.21      0.16      0.18       208\n",
      "          3       0.25      0.23      0.24       181\n",
      "          4       0.15      0.06      0.08       126\n",
      "          5       0.06      0.02      0.03       111\n",
      "\n",
      "avg / total       0.23      0.26      0.23      1160\n",
      "\n",
      "0.197876673434\n",
      "0.198014761171\n",
      "0.189099062567\n",
      "('avg_acc:', 0.25568068290328172)\n",
      "('avg_p:', 0.19726589915462353)\n",
      "('avg_r:', 0.19785385782797899)\n",
      "('avg_f:', 0.18824107248889632)\n"
     ]
    }
   ],
   "source": [
    "RF_cv(X, y, 0.99)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('X_train shape:', (4648, 440))\n",
      "('X_val shape:', (1164, 440))\n",
      "[0 1 0 ..., 3 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.286082474227\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.24      0.29      0.26       184\n",
      "          1       0.34      0.55      0.42       352\n",
      "          2       0.22      0.16      0.18       208\n",
      "          3       0.32      0.25      0.28       181\n",
      "          4       0.09      0.03      0.05       127\n",
      "          5       0.08      0.02      0.03       112\n",
      "\n",
      "avg / total       0.25      0.29      0.25      1164\n",
      "\n",
      "0.213293747341\n",
      "0.217361908233\n",
      "0.204114473088\n",
      "('X_train shape:', (4649, 448))\n",
      "('X_val shape:', (1163, 448))\n",
      "[2 1 4 ..., 0 1 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.272570937231\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.21      0.25      0.23       184\n",
      "          1       0.32      0.50      0.39       352\n",
      "          2       0.22      0.20      0.21       208\n",
      "          3       0.38      0.27      0.31       181\n",
      "          4       0.08      0.03      0.05       126\n",
      "          5       0.00      0.00      0.00       112\n",
      "\n",
      "avg / total       0.24      0.27      0.25      1163\n",
      "\n",
      "0.201027405316\n",
      "0.208617231321\n",
      "0.198012572083\n",
      "('X_train shape:', (4649, 444))\n",
      "('X_val shape:', (1163, 444))\n",
      "[1 1 1 ..., 0 1 3]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.242476354256\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.18      0.21      0.19       184\n",
      "          1       0.30      0.48      0.37       352\n",
      "          2       0.17      0.13      0.15       208\n",
      "          3       0.30      0.21      0.25       181\n",
      "          4       0.13      0.05      0.07       126\n",
      "          5       0.06      0.02      0.03       112\n",
      "\n",
      "avg / total       0.21      0.24      0.22      1163\n",
      "\n",
      "0.189459016044\n",
      "0.183252101843\n",
      "0.175193482424\n",
      "('X_train shape:', (4650, 438))\n",
      "('X_val shape:', (1162, 438))\n",
      "[4 2 1 ..., 1 1 5]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.271944922547\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.21      0.26      0.23       183\n",
      "          1       0.32      0.52      0.40       352\n",
      "          2       0.20      0.16      0.18       208\n",
      "          3       0.29      0.22      0.25       181\n",
      "          4       0.17      0.06      0.09       126\n",
      "          5       0.15      0.04      0.06       112\n",
      "\n",
      "avg / total       0.25      0.27      0.24      1162\n",
      "\n",
      "0.225962908439\n",
      "0.209735424053\n",
      "0.202107704307\n",
      "('X_train shape:', (4652, 448))\n",
      "('X_val shape:', (1160, 448))\n",
      "[1 1 3 ..., 0 2 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.270689655172\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.22      0.23      0.23       183\n",
      "          1       0.33      0.53      0.41       351\n",
      "          2       0.18      0.15      0.16       208\n",
      "          3       0.29      0.23      0.26       181\n",
      "          4       0.16      0.06      0.09       126\n",
      "          5       0.17      0.05      0.07       111\n",
      "\n",
      "avg / total       0.24      0.27      0.24      1160\n",
      "\n",
      "0.223773701699\n",
      "0.208609662272\n",
      "0.201953011037\n",
      "('avg_acc:', 0.26875286868681647)\n",
      "('avg_p:', 0.21070335576772056)\n",
      "('avg_r:', 0.20551526554446237)\n",
      "('avg_f:', 0.19627624858769982)\n"
     ]
    }
   ],
   "source": [
    "RF_cv(X, y, 0.90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('X_train shape:', (4648, 210))\n",
      "('X_val shape:', (1164, 210))\n",
      "[1 0 1 ..., 5 1 2]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.278350515464\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.18      0.22      0.20       184\n",
      "          1       0.32      0.52      0.40       352\n",
      "          2       0.26      0.21      0.23       208\n",
      "          3       0.35      0.26      0.30       181\n",
      "          4       0.09      0.03      0.05       127\n",
      "          5       0.21      0.06      0.10       112\n",
      "\n",
      "avg / total       0.26      0.28      0.25      1164\n",
      "\n",
      "0.236052756457\n",
      "0.216711147002\n",
      "0.211803294173\n",
      "('X_train shape:', (4649, 216))\n",
      "('X_val shape:', (1163, 216))\n",
      "[3 2 1 ..., 0 0 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.279449699054\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.18      0.21      0.20       184\n",
      "          1       0.32      0.53      0.40       352\n",
      "          2       0.23      0.19      0.21       208\n",
      "          3       0.38      0.25      0.30       181\n",
      "          4       0.14      0.05      0.07       126\n",
      "          5       0.28      0.07      0.11       112\n",
      "\n",
      "avg / total       0.27      0.28      0.25      1163\n",
      "\n",
      "0.254360612737\n",
      "0.217316297866\n",
      "0.215031227\n",
      "('X_train shape:', (4649, 214))\n",
      "('X_val shape:', (1163, 214))\n",
      "[2 5 3 ..., 1 3 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.263112639725\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.19      0.24      0.21       184\n",
      "          1       0.32      0.51      0.40       352\n",
      "          2       0.17      0.13      0.15       208\n",
      "          3       0.32      0.26      0.29       181\n",
      "          4       0.07      0.02      0.04       126\n",
      "          5       0.16      0.04      0.06       112\n",
      "\n",
      "avg / total       0.23      0.26      0.23      1163\n",
      "\n",
      "0.206212614975\n",
      "0.200389165059\n",
      "0.189658252739\n",
      "('X_train shape:', (4650, 210))\n",
      "('X_val shape:', (1162, 210))\n",
      "[2 0 0 ..., 1 1 0]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.26678141136\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.23      0.28      0.25       183\n",
      "          1       0.31      0.50      0.38       352\n",
      "          2       0.22      0.17      0.19       208\n",
      "          3       0.29      0.23      0.26       181\n",
      "          4       0.10      0.04      0.06       126\n",
      "          5       0.05      0.01      0.02       112\n",
      "\n",
      "avg / total       0.23      0.27      0.24      1162\n",
      "\n",
      "0.201092941365\n",
      "0.204154852096\n",
      "0.192332856427\n",
      "('X_train shape:', (4652, 215))\n",
      "('X_val shape:', (1160, 215))\n",
      "[1 1 1 ..., 2 1 4]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.280172413793\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.21      0.25      0.23       183\n",
      "          1       0.32      0.52      0.40       351\n",
      "          2       0.21      0.15      0.17       208\n",
      "          3       0.35      0.30      0.33       181\n",
      "          4       0.17      0.06      0.08       126\n",
      "          5       0.11      0.02      0.03       111\n",
      "\n",
      "avg / total       0.25      0.28      0.25      1160\n",
      "\n",
      "0.228045793\n",
      "0.217010347144\n",
      "0.206829897863\n",
      "('avg_acc:', 0.27357333587915311)\n",
      "('avg_p:', 0.2251529437066922)\n",
      "('avg_r:', 0.21111636183345825)\n",
      "('avg_f:', 0.20313110564035436)\n"
     ]
    }
   ],
   "source": [
    "RF_cv(X, y, 0.85)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('X_train shape:', (4648, 103))\n",
      "('X_val shape:', (1164, 103))\n",
      "[3 1 1 ..., 1 3 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.267182130584\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.17      0.22      0.19       184\n",
      "          1       0.31      0.52      0.39       352\n",
      "          2       0.18      0.14      0.16       208\n",
      "          3       0.43      0.28      0.34       181\n",
      "          4       0.07      0.02      0.03       127\n",
      "          5       0.16      0.04      0.07       112\n",
      "\n",
      "avg / total       0.24      0.27      0.24      1164\n",
      "\n",
      "0.220485446076\n",
      "0.20361674974\n",
      "0.195851349001\n",
      "('X_train shape:', (4649, 106))\n",
      "('X_val shape:', (1163, 106))\n",
      "[1 4 0 ..., 1 2 2]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.25709372313\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.18      0.23      0.21       184\n",
      "          1       0.30      0.48      0.37       352\n",
      "          2       0.22      0.20      0.21       208\n",
      "          3       0.33      0.19      0.24       181\n",
      "          4       0.11      0.04      0.06       126\n",
      "          5       0.21      0.04      0.07       112\n",
      "\n",
      "avg / total       0.24      0.26      0.23      1163\n",
      "\n",
      "0.225959333113\n",
      "0.198904654672\n",
      "0.19361550682\n",
      "('X_train shape:', (4649, 105))\n",
      "('X_val shape:', (1163, 105))\n",
      "[3 2 1 ..., 1 1 4]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.275150472915\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.23      0.28      0.25       184\n",
      "          1       0.32      0.49      0.39       352\n",
      "          2       0.22      0.22      0.22       208\n",
      "          3       0.36      0.24      0.29       181\n",
      "          4       0.08      0.03      0.05       126\n",
      "          5       0.19      0.04      0.07       112\n",
      "\n",
      "avg / total       0.26      0.28      0.25      1163\n",
      "\n",
      "0.233078997782\n",
      "0.216019063365\n",
      "0.209786767457\n",
      "('X_train shape:', (4650, 103))\n",
      "('X_val shape:', (1162, 103))\n",
      "[1 2 0 ..., 1 2 1]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.295180722892\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.24      0.26      0.25       183\n",
      "          1       0.34      0.54      0.41       352\n",
      "          2       0.22      0.17      0.19       208\n",
      "          3       0.41      0.32      0.36       181\n",
      "          4       0.14      0.06      0.09       126\n",
      "          5       0.12      0.04      0.05       112\n",
      "\n",
      "avg / total       0.27      0.30      0.27      1162\n",
      "\n",
      "0.243436983596\n",
      "0.231992026897\n",
      "0.226201256634\n",
      "('X_train shape:', (4652, 105))\n",
      "('X_val shape:', (1160, 105))\n",
      "[1 3 0 ..., 3 0 3]\n",
      "[0 0 0 ..., 5 5 5]\n",
      "0.302586206897\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.23      0.26      0.24       183\n",
      "          1       0.34      0.57      0.42       351\n",
      "          2       0.27      0.20      0.23       208\n",
      "          3       0.36      0.25      0.30       181\n",
      "          4       0.20      0.05      0.08       126\n",
      "          5       0.24      0.10      0.14       111\n",
      "\n",
      "avg / total       0.29      0.30      0.28      1160\n",
      "\n",
      "0.271871413641\n",
      "0.237761173016\n",
      "0.234629068803\n",
      "('avg_acc:', 0.2794386512834045)\n",
      "('avg_p:', 0.23896643484168037)\n",
      "('avg_r:', 0.21765873353785176)\n",
      "('avg_f:', 0.21201678974287025)\n"
     ]
    }
   ],
   "source": [
    "RF_cv(X, y, 0.80)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}