Ch5 - Comparative Study (ACF and Raw Features).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import ot\n",
    "import json\n",
    "import pprint\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "# Instance Based Transfer Learning\n",
    "from domain_adaptation import KMM\n",
    "from domain_adaptation import KLIEP\n",
    "from domain_adaptation import uLSIF\n",
    "from domain_adaptation import WeightedClassifier\n",
    "\n",
    "# Feature Based Transfer Learning\n",
    "from domain_adaptation import TCAClassifier\n",
    "from domain_adaptation import PCAClassifier\n",
    "from domain_adaptation import GFKClassifier\n",
    "\n",
    "# Optimal Transport Transfer Learning\n",
    "from domain_adaptation import OTClassifier\n",
    "from domain_adaptation import JDOTClassifier\n",
    "\n",
    "# Metrics for JDOT\n",
    "from domain_adaptation.metrics import MSE\n",
    "from domain_adaptation.metrics import HingeLoss\n",
    "from domain_adaptation.metrics import CategoricalCrossEntropy\n",
    "\n",
    "from utils import cval_performance\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "home_dir = \".\" # Sets home to current directory\n",
    "FEATURES = \"Raw\" # Sets which feature to use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading raw features\n"
     ]
    }
   ],
   "source": [
    "if FEATURES == 'Raw':\n",
    "    print('Loading raw features')\n",
    "    dataset = np.load('./data/cstr_rawdata.npy')\n",
    "elif FEATURES == \"ACF\":\n",
    "    print('Loading ACF features')\n",
    "    dataset = np.load('./data/cstr_acfmeandata.npy')\n",
    "\n",
    "X, y, d = dataset[:, :-4], dataset[:, -4], dataset[:, -3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table of Contents\n",
    "\n",
    "* <a href=\"#Baseline\">Baseline Evaluation</a>\n",
    "* <a href=\"#OT-Based\">Optimal Transport-Based Transfer Learning</a>\n",
    "    * <a href=\"#OTDA\">Optimal Transport Domain Adaptation</a>\n",
    "        * <a href=\"#EMD\">Exact Optimal Transport</a>\n",
    "        * <a href=\"#sinkhorn\">Sinkhorn Algorithm</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2 id=\"Baseline\">Baseline Evaluation</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = SVC(kernel='linear', max_iter=1e+6)\n",
    "per_domain_acc = []\n",
    "\n",
    "print('-' * 79)\n",
    "print('|{:^77}|'.format('Baseline'))\n",
    "print('-' * 79)\n",
    "for domain in np.unique(d).astype(int)[1:]:\n",
    "    Xt = X[np.where(d == domain)[0]]\n",
    "    yt = y[np.where(d == domain)[0]]\n",
    "    accs = cval_performance(clf, X, y, d, domain, baseline=True, normalization='normalization')\n",
    "    print(\"|{:^25}|{:^25}|{:^25}|\".format(domain,\n",
    "                                      np.round(100 * np.mean(accs), 3),\n",
    "                                      np.round(100 * np.std(accs), 3)))\n",
    "    per_domain_acc.append(np.mean(accs))\n",
    "print(\"|{:^25}|{:^25}|{:^25}|\".format('Score',\n",
    "                                  np.round(100 * np.mean(per_domain_acc), 3),\n",
    "                                  np.round(100 * np.std(per_domain_acc), 3)))\n",
    "print('-' * 79)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2 id=\"OT-Based\">Optimal Transport-Based Transfer Learning</h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3 id=\"OTDA\">Optimal Transport Domain Adaptation</h3>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h4 id=\"EMD\">Exact Optimal Transport</h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "N_grid = [None, 'max', 'median', 'log', 'loglog']\n",
    "M_grid = ['sqeuclidean', 'cityblock']\n",
    "emd_cval_result = {}\n",
    "\n",
    "# ------------------------------------- Grid Search -----------------------------------------------\n",
    "print('-' * (26 * 3) + '-')\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Normalization', 'Metric', 'Score'))\n",
    "print('-' * (26 * 3) + '-')\n",
    "\n",
    "for N in N_grid:\n",
    "    for M in M_grid:\n",
    "        clf = SVC(kernel='linear')\n",
    "        ot_clf = OTClassifier(clf=clf, ot_solver=ot.da.EMDTransport, norm=N, metric=M)\n",
    "        per_domain_acc = []\n",
    "        for domain in np.unique(d).astype(int)[1:]:\n",
    "            Xt = X[np.where(d == domain)[0]]\n",
    "            yt = y[np.where(d == domain)[0]]\n",
    "            accs = cval_performance(ot_clf, X, y, d, domain, baseline=False)\n",
    "            per_domain_acc.append(np.mean(accs))\n",
    "        emd_cval_result['{},{}'.format(str(N), str(M))] = (np.mean(per_domain_acc),\n",
    "                                                           np.std(per_domain_acc))\n",
    "        print('|{:^25}|{:^25}|{:^25}|'.format(str(N), str(M), np.mean(per_domain_acc)))\n",
    "print('-' * (26 * 3) + '-')\n",
    "\n",
    "# ------------------------------------- Best Parameters -----------------------------------------------\n",
    "pprint.pprint(emd_cval_result, width=1)\n",
    "with open(os.path.join(home_dir, './results/{}/EMD_cval.json'.format(FEATURES)), 'w') as f:\n",
    "    f.write(json.dumps(emd_cval_result))\n",
    "    \n",
    "# ------------------------------------- Evaluates for Best Parameters ---------------------------------\n",
    "print('-' * (26 * 3) + '-')\n",
    "print('|{:^77}|'.format('EMD'))\n",
    "print('-' * (26 * 3) + '-')\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Domain', 'Accuracy', 'Std Dev'))\n",
    "\n",
    "\n",
    "str_to_latex = \"\"\n",
    "clf = SVC(kernel='linear', max_iter=1e+6)\n",
    "ot_clf = OTClassifier(clf=clf, ot_solver=ot.da.EMDTransport, norm='loglog', metric='cityblock')\n",
    "\n",
    "per_domain_acc = []\n",
    "for domain in np.unique(d).astype(int)[1:]:\n",
    "    Xt = X[np.where(d == domain)[0]]\n",
    "    yt = y[np.where(d == domain)[0]]\n",
    "    accs = cval_performance(ot_clf, X, y, d, domain, baseline=False)\n",
    "    per_domain_acc.append(np.mean(accs))\n",
    "    print('|{:^25}|{:^25}|{:^25}|'.format(domain,\n",
    "                                          np.round(100 * np.mean(accs), 3),\n",
    "                                          np.round(100 * np.std(accs), 3)))\n",
    "    str_to_latex += r\"& {} $\\pm$ {}\".format(np.round(100 * np.mean(accs), 3),\n",
    "                                             np.round(100 * np.std(accs), 3))\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Score',\n",
    "                                      np.round(100 * np.mean(per_domain_acc), 3),\n",
    "                                      np.round(100 * np.std(per_domain_acc), 3)))\n",
    "str_to_latex += r\"& {}\".format(np.round(100 * np.mean(per_domain_acc), 3),\n",
    "                               np.round(100 * np.std(per_domain_acc), 3))\n",
    "print('-' * (26 * 3) + '-')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h4 id=\"sinkhorn\">Sinkhorn Algorithm</h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "M_grid = ['sqeuclidean', 'cityblock']\n",
    "R_grid = [1e-1, 1e-2, 1e-3]\n",
    "sinkhorn_cval_result = {}\n",
    "\n",
    "# ------------------------------------- Grid Search -----------------------------------------------\n",
    "print('-' * (26 * 3) + '-')\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Normalization', 'Entropic Reg', 'Score'))\n",
    "print('-' * (26 * 3) + '-')\n",
    "for N in M_grid:\n",
    "    for R in R_grid:\n",
    "        clf = SVC(kernel='linear', max_iter=1e+6)\n",
    "        ot_clf = OTClassifier(clf=clf, ot_solver=ot.da.SinkhornTransport, norm='max', reg_e=R, metric=N)\n",
    "        per_domain_acc = []\n",
    "        for domain in np.unique(d).astype(int)[1:]:\n",
    "            Xt = X[np.where(d == domain)[0]]\n",
    "            yt = y[np.where(d == domain)[0]]\n",
    "            accs = cval_performance(ot_clf, X, y, d, domain, baseline=False)\n",
    "            per_domain_acc.append(np.mean(accs))\n",
    "        sinkhorn_cval_result['{},{}'.format(N, R)] = (np.mean(per_domain_acc),\n",
    "                                                      np.std(per_domain_acc))\n",
    "        print('|{:^25}|{:^25}|{:^25}|'.format(N, R, np.mean(per_domain_acc)))\n",
    "print('-' * (26 * 3) + '-')\n",
    "\n",
    "# ------------------------------------- Saves Results ---------------------------------------------\n",
    "pprint.pprint(sinkhorn_cval_result, width=1)\n",
    "with open(os.path.join(home_dir, './results/{}/Sinkhorn_cval.json'.format(FEATURES)), 'w') as f:\n",
    "    f.write(json.dumps(sinkhorn_cval_result))\n",
    "\n",
    "# ------------------------------------- Runs for best parameters ----------------------------------\n",
    "M_grid = ['sqeuclidean', 'cityblock']\n",
    "R_grid = [1e-1, 1e-2, 1e-3]\n",
    "sinkhorn_cval_result = {}\n",
    "\n",
    "print('-' * (26 * 3) + '-')\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Domain', 'Mean Acc', 'Std'))\n",
    "print('-' * (26 * 3) + '-')\n",
    "\n",
    "clf = SVC(kernel='linear', max_iter=1e+6)\n",
    "ot_clf = OTClassifier(clf=clf, ot_solver=ot.da.SinkhornTransport, norm='max', reg_e=1e-3, metric='cityblock')\n",
    "per_domain_acc = []\n",
    "for domain in np.unique(d).astype(int)[1:]:\n",
    "    Xt = X[np.where(d == domain)[0]]\n",
    "    yt = y[np.where(d == domain)[0]]\n",
    "    accs = cval_performance(ot_clf, X, y, d, domain, baseline=False)\n",
    "    per_domain_acc.append(np.mean(accs))\n",
    "    print('|{:^25}|{:^25}|{:^25}|'.format(domain, np.mean(accs), np.std(accs)))\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Score', np.mean(per_domain_acc), np.std(per_domain_acc)))\n",
    "print('-' * (26 * 3) + '-')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h4 id=\"sinkhornl1l2\">Sinkhorn \\(\\ell_{1}-\\ell_{2}\\)</h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sinkhorn_l1l2_cval_result = {}\n",
    "R_grid = [1e-1, 1e-2, 1e-3]\n",
    "E_grid = [1e-1, 1e-2, 1e-3]\n",
    "\n",
    "# ------------------------------------- Grid Search -----------------------------------------------\n",
    "print('-' * (26 * 3) + '-')\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Entropic Reg', 'Class Reg', 'Score'))\n",
    "print('-' * (26 * 3) + '-')\n",
    "\n",
    "for R in R_grid:\n",
    "    for E in E_grid:\n",
    "        clf = SVC(kernel='linear', max_iter=1e+6)\n",
    "        ot_clf = OTClassifier(clf=clf,\n",
    "                              ot_solver=ot.da.SinkhornL1l2Transport,\n",
    "                              norm='max',\n",
    "                              reg_e=R,\n",
    "                              reg_cl=E,\n",
    "                              metric='cityblock')\n",
    "        per_domain_acc = []\n",
    "        for domain in np.unique(d).astype(int)[1:]:\n",
    "            Xt = X[np.where(d == domain)[0]]\n",
    "            yt = y[np.where(d == domain)[0]]\n",
    "            try: \n",
    "                accs = cval_performance(ot_clf, X, y, d, domain, baseline=False)\n",
    "            except:\n",
    "                accs = [-1]\n",
    "            per_domain_acc.append(np.mean(accs))\n",
    "        sinkhorn_l1l2_cval_result['{},{}'.format(R, E)] = (np.mean(per_domain_acc),\n",
    "                                                           np.std(per_domain_acc))\n",
    "        print('|{:^25}|{:^25}|{:^25}|'.format(R, E, np.mean(per_domain_acc)))\n",
    "print('-' * (26 * 3) + '-')\n",
    "\n",
    "# ------------------------------------- Saves Results ---------------------------------------------\n",
    "pprint.pprint(sinkhorn_l1l2_cval_result, width=1)\n",
    "with open(os.path.join(home_dir, './results/{}/sinkhorn_l1l2_cval.json'.format(FEATURES)), 'w') as f:\n",
    "    f.write(json.dumps(sinkhorn_l1l2_cval_result))\n",
    "    \n",
    "# ------------------------------------- Runs for best parameters ----------------------------------\n",
    "M_grid = ['sqeuclidean', 'cityblock']\n",
    "R_grid = [1e-1, 1e-2, 1e-3]\n",
    "sinkhorn_cval_result = {}\n",
    "\n",
    "print('-' * (26 * 3) + '-')\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Domain', 'Mean Acc', 'Std'))\n",
    "print('-' * (26 * 3) + '-')\n",
    "\n",
    "clf = SVC(kernel='linear', max_iter=1e+6)\n",
    "ot_clf = OTClassifier(clf=clf,\n",
    "                      ot_solver=ot.da.SinkhornL1l2Transport,\n",
    "                      norm='max',\n",
    "                      reg_e=1e-3,\n",
    "                      reg_cl=1e-3,\n",
    "                      metric='cityblock')\n",
    "per_domain_acc = []\n",
    "for domain in np.unique(d).astype(int)[1:]:\n",
    "    Xt = X[np.where(d == domain)[0]]\n",
    "    yt = y[np.where(d == domain)[0]]\n",
    "    accs = cval_performance(ot_clf, X, y, d, domain, baseline=False)\n",
    "    per_domain_acc.append(np.mean(accs))\n",
    "    print('|{:^25}|{:^25}|{:^25}|'.format(domain, np.mean(accs), np.std(accs)))\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Score', np.mean(per_domain_acc), np.std(per_domain_acc)))\n",
    "print('-' * (26 * 3) + '-')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h4 id=\"sinkhornlpl1\">Sinkhorn \\(\\ell_{p}-\\ell_{1}\\)</h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sinkhorn_lpl1_cval_result = {}\n",
    "R_grid = [1e-1, 1e-2, 1e-3]\n",
    "E_grid = [1e-1, 1e-2, 1e-3]\n",
    "# ------------------------------------- Grid Search -----------------------------------------------\n",
    "print('-' * (26 * 4) + '-')\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Entropic Reg', 'Class Reg', 'Score'))\n",
    "print('-' * (26 * 4) + '-')\n",
    "\n",
    "for R in R_grid:\n",
    "    for E in E_grid:\n",
    "        clf = SVC(kernel='linear', max_iter=1e+6)\n",
    "        ot_clf = OTClassifier(clf=clf,\n",
    "                            ot_solver=ot.da.SinkhornLpl1Transport,\n",
    "                            norm='max',\n",
    "                            reg_e=R,\n",
    "                            reg_cl=E)\n",
    "        per_domain_acc = []\n",
    "        for domain in np.unique(d).astype(int)[1:]:\n",
    "            Xt = X[np.where(d == domain)[0]]\n",
    "            yt = y[np.where(d == domain)[0]]\n",
    "            accs = cval_performance(ot_clf, X, y, d, domain, baseline=False)\n",
    "            per_domain_acc.append(np.mean(accs))\n",
    "        sinkhorn_lpl1_cval_result['{},{}'.format(R, E)] = (np.mean(per_domain_acc),\n",
    "                                                           np.std(per_domain_acc))\n",
    "        print('|{:^25}|{:^25}|{:^25}|'.format(R, E, np.mean(per_domain_acc)))\n",
    "print('-' * (26 * 4) + '-')\n",
    "\n",
    "# ------------------------------------- Saves Results ---------------------------------------------\n",
    "pprint.pprint(sinkhorn_lpl1_cval_result, width=1)\n",
    "with open(os.path.join(home_dir, './results/{}/sinkhorn_lpl1_cval.json'.format(FEATURES)), 'w') as f:\n",
    "    f.write(json.dumps(sinkhorn_lpl1_cval_result))\n",
    "\n",
    "# ------------------------------------- Runs for best parameters ----------------------------------\n",
    "print('-' * (26 * 3) + '-')\n",
    "print('|{:^77}|'.format('Sinkhorn LpL1'))\n",
    "print('-' * (26 * 3) + '-')\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Domain', 'Accuracy', 'Std Dev'))\n",
    "\n",
    "clf = SVC(kernel='linear', max_iter=1e+6)\n",
    "ot_clf = OTClassifier(clf=clf, ot_solver=ot.da.SinkhornLpl1Transport,\n",
    "                      norm='max',\n",
    "                      metric='cityblock',\n",
    "                      reg_e=1e-3,\n",
    "                      reg_cl=1e-2)\n",
    "\n",
    "per_domain_acc = []\n",
    "for domain in np.unique(d).astype(int)[1:]:\n",
    "    Xt = X[np.where(d == domain)[0]]\n",
    "    yt = y[np.where(d == domain)[0]]\n",
    "    accs = cval_performance(ot_clf, X, y, d, domain,\n",
    "                            index_crossval=crossval_index, baseline=False)\n",
    "    per_domain_acc.append(np.mean(accs))\n",
    "    print('|{:^25}|{:^25}|{:^25}|'.format(domain,\n",
    "                                          np.round(100 * np.mean(accs), 3),\n",
    "                                          np.round(100 * np.std(accs), 3)))\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Score',\n",
    "                                      np.round(100 * np.mean(per_domain_acc), 3),\n",
    "                                      np.round(100 * np.std(per_domain_acc), 3)))\n",
    "print('-' * (26 * 3) + '-')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3 id=\"MP\">Mapping Estimation (Monge Problem)</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MP_cval_result = {}\n",
    "K_grid = ['linear', 'gaussian']\n",
    "M_grid = [1e+0, 1e-1, 1e-2, 1e-3]\n",
    "E_grid = [1e-3, 1e-2, 1e-1, 1e+0]\n",
    "B_grid = [True]\n",
    "\n",
    "# ------------------------------------- Grid Search -----------------------------------------------\n",
    "print('-' * (18 * 5) + '-')\n",
    "print('|{:^17}|{:^17}|{:^17}|{:^17}|{:^17}|'.format('Kernel',\n",
    "                                             'Loss Weight',\n",
    "                                             'Linear Map Reg',\n",
    "                                             'Fit bias',\n",
    "                                             'Score'))\n",
    "print('-' * (18 * 5) + '-')\n",
    "\n",
    "for K in K_grid:\n",
    "    for mu in M_grid:\n",
    "        for eta in E_grid:\n",
    "            for bias in B_grid:\n",
    "                clf = SVC(kernel='linear', max_iter=1e+6)\n",
    "                ot_clf = OTClassifier(clf=clf,\n",
    "                                    ot_solver=ot.da.MappingTransport,\n",
    "                                    norm='max',\n",
    "                                    max_iter=3,\n",
    "                                    mu=mu,\n",
    "                                    eta=eta,\n",
    "                                    bias=bias,\n",
    "                                    kernel=K, metric='cityblock')\n",
    "                per_domain_acc = []\n",
    "                for domain in np.unique(d).astype(int)[1:]:\n",
    "                    Xt = X[np.where(d == domain)[0]]\n",
    "                    yt = y[np.where(d == domain)[0]]\n",
    "                    accs = cval_performance(ot_clf, X, y, d, domain, baseline=False)\n",
    "                    per_domain_acc.append(np.mean(accs))\n",
    "                MP_cval_result['{},{},{},{}'.format(K,\n",
    "                                                    mu,\n",
    "                                                    eta,\n",
    "                                                    bias)] = (np.mean(per_domain_acc),\n",
    "                                                              np.std(per_domain_acc))\n",
    "                print('|{:^17}|{:^17}|{:^17}|{:^17}|{:^17}|'.format(K, mu, eta, bias,\n",
    "                                                             np.round(np.mean(per_domain_acc), 6)))\n",
    "print('-' * (18 * 5) + '-')\n",
    "# ------------------------------------- Saves Results ---------------------------------------------\n",
    "pprint.pprint(MP_cval_result, width=1)\n",
    "with open('./results/MP_cval.json', 'w') as f:\n",
    "    f.write(json.dumps(MP_cval_result))\n",
    "    \n",
    "# ------------------------------------- Runs for best parameters ----------------------------------\n",
    "print('-' * (26 * 3) + '-')\n",
    "print('|{:^77}|'.format('Mapping Estimation'))\n",
    "print('-' * (26 * 3) + '-')\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Domain', 'Accuracy', 'Std Dev'))\n",
    "\n",
    "\n",
    "str_to_latex = \"\"\n",
    "clf = SVC(kernel='linear', max_iter=1e+6)\n",
    "ot_clf = OTClassifier(clf=clf,\n",
    "                      ot_solver=ot.da.MappingTransport,\n",
    "                      norm='max',\n",
    "                      max_iter=3,\n",
    "                      mu=0.1,\n",
    "                      eta=0.001,\n",
    "                      bias=True,\n",
    "                      kernel='linear',\n",
    "                      metric='sqeuclidean')\n",
    "\n",
    "per_domain_acc = []\n",
    "for domain in np.unique(d).astype(int)[1:]:\n",
    "    Xt = X[np.where(d == domain)[0]]\n",
    "    yt = y[np.where(d == domain)[0]]\n",
    "    accs = cval_performance(ot_clf, X, y, d, domain, baseline=False)\n",
    "    per_domain_acc.append(np.mean(accs))\n",
    "    print('|{:^25}|{:^25}|{:^25}|'.format(domain,\n",
    "                                          np.round(100 * np.mean(accs), 3),\n",
    "                                          np.round(100 * np.std(accs), 3)))\n",
    "    str_to_latex += r\"& {} $\\pm$ {}\".format(np.round(100 * np.mean(accs), 3),\n",
    "                                             np.round(100 * np.std(accs), 3))\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Score',\n",
    "                                      np.round(100 * np.mean(per_domain_acc), 3),\n",
    "                                      np.round(100 * np.std(per_domain_acc), 3)))\n",
    "str_to_latex += r\"& {}\".format(np.round(100 * np.mean(per_domain_acc), 3),\n",
    "                               np.round(100 * np.std(per_domain_acc), 3))\n",
    "print('-' * (26 * 3) + '-')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3 id=\"JDOT\">Joint Distribution Optimal Transport</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "R_grid = [0.0, 1e-2, 1e-1, 1, 1e+1]\n",
    "L_grid = [hinge_loss, cat_loss, mse_loss]\n",
    "A_grid = [1e-1, 1, 1e+1, 1e+2]\n",
    "jdot_cval_result = {}\n",
    "\n",
    "\n",
    "# ------------------------------------- Grid Search -----------------------------------------------\n",
    "print('-' * (26 * 4) + '-')\n",
    "print('|{:^25}|{:^25}|{:^25}|{:^25}|'.format('Alpha', 'Loss Fn', 'Entropic Reg', 'Score'))\n",
    "print('-' * (26 * 4) + '-')\n",
    "\n",
    "for A in A_grid:\n",
    "    for L in L_grid:\n",
    "        for R in R_grid:\n",
    "            method = 'sinkhorn' if R > 0.0 else 'emd'\n",
    "            clf = SVC(kernel='linear', max_iter=1e+6)\n",
    "            jdot_clf = JDOTClassifier(clf=clf,\n",
    "                                      method=method,\n",
    "                                      reg=R,\n",
    "                                      alpha=A,\n",
    "                                      loss=L,\n",
    "                                      numItermax=5,\n",
    "                                      metric='cityblock')\n",
    "            per_domain_acc = []\n",
    "            for domain in np.unique(d).astype(int)[1:]:\n",
    "                Xt = X[np.where(d == domain)[0]]\n",
    "                yt = y[np.where(d == domain)[0]]\n",
    "                try:\n",
    "                    accs = cval_performance(jdot_clf, X, y, d, domain,\n",
    "                                            baseline=False)\n",
    "                except:\n",
    "                    accs = [-1]\n",
    "                per_domain_acc.append(np.mean(accs))\n",
    "            jdot_cval_result['{},{},{}'.format(A, L, R)] = (np.mean(per_domain_acc),\n",
    "                                                            np.std(per_domain_acc))\n",
    "            print('|{:^25}|{:^25}|{:^25}|{:^25}|'.format(A, L, R, np.mean(per_domain_acc)))\n",
    "print('-' * (26 * 4) + '-')\n",
    "\n",
    "# ------------------------------------- Saves Results ---------------------------------------------\n",
    "pprint.pprint(jdot_cval_result, width=1)\n",
    "with open(os.path.join(home_dir, './results/{}/JDOT_cval.json'.format(FEATURES)), 'w') as f:\n",
    "    f.write(json.dumps(jdot_cval_result))\n",
    "\n",
    "# ------------------------------------- Runs for best parameters ----------------------------------\n",
    "print('-' * (26 * 3) + '-')\n",
    "print('|{:^77}|'.format('JDOT'))\n",
    "print('-' * (26 * 3) + '-')\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Domain', 'Accuracy', 'Std Dev'))\n",
    "\n",
    "clf = SVC(kernel='linear', max_iter=1e+6)\n",
    "ot_clf = JDOTClassifier(clf=clf, loss=hinge_loss, alpha=10, method='emd', numItermax=5,\n",
    "                        metric='cityblock')\n",
    "str_to_latex = \"\"\n",
    "\n",
    "per_domain_acc = []\n",
    "for domain in np.unique(d).astype(int)[1:]:\n",
    "    Xt = X[np.where(d == domain)[0]]\n",
    "    yt = y[np.where(d == domain)[0]]\n",
    "    accs = cval_performance(ot_clf, X, y, d, domain, baseline=False)\n",
    "    per_domain_acc.append(np.mean(accs))\n",
    "    print('|{:^25}|{:^25}|{:^25}|'.format(domain,\n",
    "                                          np.round(100 * np.mean(accs), 3),\n",
    "                                          np.round(100 * np.std(accs), 3)))\n",
    "    str_to_latex += \"& {} $\\pm$ {}\".format(np.round(100 * np.mean(accs), 3),\n",
    "                                           np.round(100 * np.std(accs), 3))\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Score',\n",
    "                                      np.round(100 * np.mean(per_domain_acc), 3),\n",
    "                                      np.round(100 * np.std(per_domain_acc), 3)))\n",
    "str_to_latex += \"& {} $\\pm$ {}\".format(np.round(100 * np.mean(per_domain_acc), 3),\n",
    "                                        np.round(100 * np.std(per_domain_acc), 3))\n",
    "print('-' * (26 * 3) + '-')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2 id=\"Feat-Based\">Feature-Based Transfer Learning</h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3 id=\"PCA\">Principal Component Analysis</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if FEATURES == \"ACF\":\n",
    "    D_grid = [1, 2, 5, 10, 13]\n",
    "elif FEATURES == \"Raw\":\n",
    "    D_grid = [2, 5, 10, 15, 20, 25, 30, 100, 200, 300]\n",
    "pca_cval_result = {}\n",
    "clf = SVC(kernel='linear', max_iter=1e+6)\n",
    "\n",
    "# ------------------------------------- Grid Search -----------------------------------------------\n",
    "print('-' * (21 * 2) + '-')\n",
    "print('|{:^20}|{:^20}|'.format('N Dimensions', 'Score'))\n",
    "print('-' * (21 * 2) + '-')\n",
    "for ndims in D_grid:\n",
    "    pca_clf = PCAClassifier(clf=clf, n_components=ndims)\n",
    "    per_domain_acc = []\n",
    "    for domain in np.unique(d).astype(int)[1:]:\n",
    "        Xt = X[np.where(d == domain)[0]]\n",
    "        yt = y[np.where(d == domain)[0]]\n",
    "        accs = cval_performance(pca_clf, X, y, d, domain, baseline=False)\n",
    "        per_domain_acc.append(np.mean(accs))\n",
    "    pca_cval_result['{}'.format(ndims)] = (np.mean(per_domain_acc),\n",
    "                                       np.std(per_domain_acc))\n",
    "    print('|{:^20}|{:^20}|'.format(ndims, np.mean(per_domain_acc)))\n",
    "print('-' * (21 * 2) + '-')\n",
    "\n",
    "# ------------------------------------- Saves Results ---------------------------------------------\n",
    "pprint.pprint(pca_cval_result, width=1)\n",
    "with open('./results/{}/PCA_cval.json'.format(FEATURES), 'w') as f:\n",
    "    f.write(json.dumps(pca_cval_result))\n",
    "\n",
    "# ------------------------------------- Runs for best parameters ----------------------------------\n",
    "clf = SVC(kernel='linear', max_iter=1e+6)\n",
    "pca_clf = PCAClassifier(clf=clf, n_components=25)\n",
    "str_to_latex = \"\"\n",
    "\n",
    "print('-' * (26 * 3) + '-')\n",
    "print('|{:^77}|'.format('PCA'))\n",
    "print('-' * (26 * 3) + '-')\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Domain', 'Accuracy', 'Std Dev'))\n",
    "\n",
    "per_domain_acc = []\n",
    "for domain in np.unique(d).astype(int)[1:]:\n",
    "    Xt = X[np.where(d == domain)[0]]\n",
    "    yt = y[np.where(d == domain)[0]]\n",
    "    accs = cval_performance(pca_clf, X, y, d, domain, baseline=False)\n",
    "    per_domain_acc.append(np.mean(accs))\n",
    "    print('|{:^25}|{:^25}|{:^25}|'.format(domain,\n",
    "                                          np.round(100 * np.mean(accs), 3),\n",
    "                                          np.round(100 * np.std(accs), 3)))\n",
    "    str_to_latex += r\"& {} $\\\\pm$ {}\".format(np.round(100 * np.mean(accs), 3),\n",
    "                                            np.round(100 * np.std(accs), 3))\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Score',\n",
    "                                      np.round(100 * np.mean(per_domain_acc), 3),\n",
    "                                      np.round(100 * np.std(per_domain_acc), 3)))\n",
    "str_to_latex += r\"& {}\".format(np.round(100 * np.mean(per_domain_acc), 3))\n",
    "print('-' * (26 * 3) + '-')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3 id=\"TCA\">Transfer Component Analysis</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "D_grid = [1, 2, 5, 10, 13]\n",
    "K_grid = ['linear', 'rbf']\n",
    "M_grid = [1e-2, 1e-1, 1, 1e+1, 1e+2]\n",
    "G_grid = [1e-3, 1e-2, 1e-1, 1e0]\n",
    "N_grid = ['none', 'scale', 'std']\n",
    "\n",
    "tca_cval_result = {}\n",
    "clf = SVC(kernel='linear', max_iter=1e+6)\n",
    "# ------------------------------------- Grid Search -----------------------------------------------\n",
    "print('-' * (18 * 6) + '-')\n",
    "print('|{:^17}|{:^17}|{:^17}|{:^17}|{:^17}|{:^17}|'.format('Ndim',\n",
    "                                                           'Kernel',\n",
    "                                                           'Mu',\n",
    "                                                           'Kernel Param',\n",
    "                                                           'Norm',\n",
    "                                                           'Score'))\n",
    "print('-' * (18 * 6) + '-')\n",
    "for ndims in D_grid:\n",
    "    for K in K_grid:\n",
    "        for M in M_grid:\n",
    "            for G in G_grid:\n",
    "                for N in N_grid:\n",
    "                    tca_clf = TCAClassifier(clf=clf, n_components=ndims, kernel=K, mu=M, gamma=G,\n",
    "                                            normalize_projections=N)\n",
    "                    per_domain_acc = []\n",
    "                    for domain in np.unique(d).astype(int)[1:]:\n",
    "                        Xt = X[np.where(d == domain)[0]]\n",
    "                        yt = y[np.where(d == domain)[0]]\n",
    "                        accs = cval_performance(tca_clf, X, y, d, domain, baseline=False)\n",
    "                        per_domain_acc.append(np.mean(accs))\n",
    "                    tca_cval_result['{},{},{},{},{}'.format(ndims,\n",
    "                                                            K, M, G, N)] = (np.mean(per_domain_acc),\n",
    "                                                                            np.std(per_domain_acc))\n",
    "                    print('|{:^17}|{:^17}|{:^17}|{:^17}|{:^17}|{:^17}|'.format(ndims, K, M, G, N,\n",
    "                                                                               np.round(np.mean(per_domain_acc), 8)))\n",
    "print('-' * (17 * 6) + '-')\n",
    "\n",
    "# ------------------------------------- Saves Results ---------------------------------------------\n",
    "pprint.pprint(tca_cval_result, width=1)\n",
    "with open('./results/{}/TCA_cval.json'.format(FEATURES), 'w') as f:\n",
    "    f.write(json.dumps(tca_cval_result))\n",
    "\n",
    "# ------------------------------------- Runs for best parameters ----------------------------------\n",
    "clf = SVC(kernel='linear', max_iter=1e+6)\n",
    "tca_clf = TCAClassifier(clf=clf, n_components=300, kernel='rbf', mu=10.0, gamma=None,\n",
    "                        normalize_projections='none')\n",
    "str_to_latex = \"\"\n",
    "\n",
    "print('-' * (26 * 3) + '-')\n",
    "print('|{:^77}|'.format('TCA'))\n",
    "print('-' * (26 * 3) + '-')\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Domain', 'Mean Accuracy', 'Std Accuracy'))\n",
    "print('-' * (26 * 3) + '-')\n",
    "per_domain_acc = []\n",
    "for domain in np.unique(d).astype(int)[1:]:\n",
    "    Xt = X[np.where(d == domain)[0]]\n",
    "    yt = y[np.where(d == domain)[0]]\n",
    "    accs = cval_performance(tca_clf, X, y, d, domain, baseline=False, normalization=\"normalization\")\n",
    "    mean_acc = np.round(100 * np.mean(accs), 3)\n",
    "    std_acc = np.round(100 * np.std(accs), 3)\n",
    "    print('|{:^25}|{:^25}|{:^25}|'.format(domain, mean_acc, std_acc))\n",
    "    str_to_latex += \"{} $\\pm$ {}\".format(mean_acc, std_acc)\n",
    "    per_domain_acc.append(np.mean(accs))\n",
    "mean_acc = np.round(100 * np.mean(per_domain_acc), 3)\n",
    "std_acc = np.round(100 * np.std(per_domain_acc), 3)\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('score', mean_acc, std_acc))\n",
    "str_to_latex += \"{}\".format(mean_acc)\n",
    "    \n",
    "print('-' * (26 * 3) + '-')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3 id=\"GFK\">Geodesic Flow Kernel</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if FEATURES == \"ACF\":\n",
    "    D_grid = [1, 2, 5, 10, 13]\n",
    "elif FEATURES == \"Raw\":\n",
    "    D_grid = [2, 5, 10, 15, 20, 25, 30, 100, 200, 300]\n",
    "P_grid = ['pca', 'pls']\n",
    "\n",
    "gfk_cval_result = {}\n",
    "clf = SVC(kernel='linear', max_iter=1e+6)\n",
    "# ------------------------------------- Grid Search -----------------------------------------------\n",
    "print('-' * (26 * 3) + '-')\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Ndim', 'Projection', 'Score'))\n",
    "print('-' * (26 * 3) + '-')\n",
    "for ndims in D_grid:\n",
    "    for P in P_grid:\n",
    "        gfk_clf = GFKClassifier(clf=clf, n_components=ndims, projection=P)\n",
    "        per_domain_acc = []\n",
    "        for domain in np.unique(d).astype(int)[1:]:\n",
    "            Xt = X[np.where(d == domain)[0]]\n",
    "            yt = y[np.where(d == domain)[0]]\n",
    "            try:\n",
    "                accs = cval_performance(gfk_clf, X, y, d, domain, baseline=False)\n",
    "            except:\n",
    "                accs = [-1]\n",
    "            per_domain_acc.append(np.mean(accs))\n",
    "        gfk_cval_result['{},{}'.format(ndims, P)] = (np.mean(per_domain_acc), np.std(per_domain_acc))\n",
    "        print('|{:^25}|{:^25}|{:^25}|'.format(ndims, P, np.round(np.mean(per_domain_acc), 8)))\n",
    "print('-' * (26 * 3) + '-')\n",
    "\n",
    "# ------------------------------------- Saves Results ---------------------------------------------\n",
    "pprint.pprint(gfk_cval_result, width=1)\n",
    "with open('./results/GFK_cval.json', 'w') as f:\n",
    "    f.write(json.dumps(gfk_cval_result))\n",
    "    \n",
    "\n",
    "# ------------------------------------- Runs for best parameters ----------------------------------\n",
    "str_to_latex = \"\"\n",
    "\n",
    "print('-' * (26 * 3) + '-')\n",
    "print('|{:^77}|'.format('GFK'))\n",
    "print('-' * (26 * 3) + '-')\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Domain', 'Accuracy', 'Std Dev'))\n",
    "\n",
    "clf = SVC(kernel='linear', max_iter=1e+6)\n",
    "gfk_clf = GFKClassifier(clf=clf, n_components=100, projection='pca')\n",
    "\n",
    "per_domain_acc = []\n",
    "for domain in np.unique(d).astype(int)[1:]:\n",
    "    Xt = X[np.where(d == domain)[0]]\n",
    "    yt = y[np.where(d == domain)[0]]\n",
    "    accs = cval_performance(gfk_clf, X, y, d, domain, baseline=False, normalization='normalization')\n",
    "    per_domain_acc.append(np.mean(accs))\n",
    "    print('|{:^25}|{:^25}|{:^25}|'.format(domain,\n",
    "                                          np.round(100 * np.mean(accs), 3),\n",
    "                                          np.round(100 * np.std(accs), 3)))\n",
    "    str_to_latex += r\"& {} $\\pm$ {}\".format(np.round(100 * np.mean(accs), 3),\n",
    "                                           np.round(100 * np.std(accs), 3))\n",
    "print('|{:^25}|{:^25}|{:^25}|'.format('Score',\n",
    "                                      np.round(100 * np.mean(per_domain_acc), 3),\n",
    "                                      np.round(100 * np.std(per_domain_acc), 3)))\n",
    "str_to_latex += r\"& {}\".format(np.round(100 * np.mean(per_domain_acc), 3))\n",
    "print('-' * (26 * 3) + '-')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2 id=\"Instance-Based\">Instance-Based Transfer Learning</h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3 id=\"KLIEP\">Kullback-Leibler Importance Estimation Procedure</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "K_grid = ['rbf']\n",
    "BV_grid = ['None', 'Xs', 'Xt', 'XsXt']\n",
    "G_grid = [[1e-3], [1e-2], [1e-1], None]\n",
    "E_grid = [1e-1, 1e-2, 1e-3, 1e-4]\n",
    "kliep_cval_result = {}\n",
    "\n",
    "print('-' * (18 * 6) + '-')\n",
    "print('|{:^17}|{:^17}|{:^17}|{:^17}|{:^17}|{:^17}|'.format('C', 'Kernel', 'Basis Vectors', 'G', 'E', 'Score'))\n",
    "print('-' * (18 * 6) + '-')\n",
    "for C in [1, 15, 25]:\n",
    "    for K in K_grid:\n",
    "        for BV_str in BV_grid:\n",
    "            for G in G_grid:\n",
    "                for E in E_grid:\n",
    "                    if K == 'linear' and G is not None:\n",
    "                        pass\n",
    "                    else:\n",
    "                        clf = SVC(C=C, kernel='linear')\n",
    "                        if BV_str == 'None':\n",
    "                            BV = None\n",
    "                        elif BV_str == 'Xs':\n",
    "                            BV = Xs\n",
    "                        elif BV_str == 'Xt':\n",
    "                            BV = Xt\n",
    "                        elif BV_str == 'XsXt':\n",
    "                            BV = np.concatenate([Xs, Xt], axis=0)\n",
    "                        wclf = WeightedClassifier(clf,\n",
    "                                                  KLIEP,\n",
    "                                                  kernel=K,\n",
    "                                                  kernel_param=G,\n",
    "                                                  basis_vectors=BV,\n",
    "                                                  epsilon=E)\n",
    "                        per_domain_acc = []\n",
    "                        for domain in np.unique(d).astype(int)[1:]:\n",
    "                            accs = cval_performance(wclf, X, y, d, domain, baseline=False)\n",
    "                            per_domain_acc.append(np.mean(accs))\n",
    "                        kliep_cval_result['{},{},{},{},{}'.format(C,\n",
    "                                                                  K,\n",
    "                                                                  BV_str,\n",
    "                                                                  G,\n",
    "                                                                  E)] = (np.mean(per_domain_acc),\n",
    "                                                                         np.std(per_domain_acc))\n",
    "                        print('|{:^17}|{:^17}|{:^17}'\n",
    "                              '|{:^17}|{:^17}|{:^17}|'.format(C, K, BV_str, str(G),\n",
    "                                                              E, np.round(np.mean(per_domain_acc), 7)))\n",
    "print('-' * (18 * 6) + '-')\n",
    "\n",
    "pprint.pprint(kliep_cval_result, width=1)\n",
    "with open('./results/KLIEP_cval.json', 'w') as f:\n",
    "    f.write(json.dumps(kliep_cval_result))\n",
    "    \n",
    "clf = SVC(C=1, kernel='linear')\n",
    "wclf = WeightedClassifier(clf, KLIEP, kernel='rbf', kernel_param=None, basis_vectors=None, epsilon=1e-3)\n",
    "print('-' * 79)\n",
    "print('|{:^77}|'.format('Baseline'))\n",
    "print('-' * 79)\n",
    "print(\"|{:^25}|{:^25}|{:^25}|\".format(\"Domain\", \"Mean Accuracy\", \"Std Deviation\"))\n",
    "print('-' * 79)\n",
    "per_domain_acc = []\n",
    "for domain in np.unique(d).astype(int)[1:]:\n",
    "    Xt = X[np.where(d == domain)[0]]\n",
    "    yt = y[np.where(d == domain)[0]]\n",
    "    accs = cval_performance(clf, Xs, ys, Xt, yt, n_folds=5, baseline=True)\n",
    "    per_domain_acc.append(np.mean(accs))\n",
    "    print(\"|{:^25}|{:^25}|{:^25}|\".format(domain, np.mean(accs), np.std(accs)))\n",
    "print(\"|{:^25}|{:^25}|{:^25}|\".format('-', np.mean(per_domain_acc), np.std(per_domain_acc)))\n",
    "print('-' * 79)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3 id=\"uLSIF\">unconstrained Least Squares Importance Fitting</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "K_grid = ['rbf']\n",
    "G_grid = [[1e-3], [1e-2], [1e-1], None]\n",
    "R_grid = [1e-1, 1e-2, 1e-3, 1e-9]\n",
    "\n",
    "ulsif_cval_result = {}\n",
    "print('-' * (21 * 5) + '-')\n",
    "print('|{:^20}|{:^20}|{:^20}|{:^20}|{:^20}|'.format('C', 'Kernel', 'G', 'R', 'Score'))\n",
    "print('-' * (21 * 5) + '-')\n",
    "for C in [10, 15, 25]:\n",
    "    clf = SVC(C=C, kernel='linear')\n",
    "    for K in K_grid:\n",
    "        for G in G_grid:\n",
    "            for R in R_grid:\n",
    "                if K == 'linear' and G is not None:\n",
    "                    pass\n",
    "                else:\n",
    "                    wclf = WeightedClassifier(clf, uLSIF, kernel=K, kernel_param=G, basis_vectors=None, reg=R)\n",
    "                    per_domain_acc = []\n",
    "                    for domain in np.unique(d).astype(int)[1:]:\n",
    "                        accs = cval_performance(wclf, X, y, d, domain, baseline=False)\n",
    "                        per_domain_acc.append(np.mean(accs))\n",
    "                    ulsif_cval_result['{},{},{},{}'.format(C, K, G, R)] = (np.mean(per_domain_acc),\n",
    "                                                                           np.std(per_domain_acc))\n",
    "                    print('|{:^20}|{:^20}|{:^20}|{:^20}|{:^20}|'.format(C, K, str(G),\n",
    "                                                                        R, np.mean(per_domain_acc)))\n",
    "                    \n",
    "pprint.pprint(ulsif_cval_result, width=1)\n",
    "with open('./results/uLSIF_cval.json', 'w') as f:\n",
    "    f.write(json.dumps(ulsif_cval_result))\n",
    "    \n",
    "clf = SVC(C=1, kernel='linear')\n",
    "wclf = WeightedClassifier(clf, uLSIF, kernel='linear', kernel_param=[10], basis_vectors=None, reg=1e-1)\n",
    "print('-' * 79)\n",
    "print('|{:^77}|'.format('Baseline'))\n",
    "print('-' * 79)\n",
    "print(\"|{:^25}|{:^25}|{:^25}|\".format(\"Domain\", \"Mean Accuracy\", \"Std Deviation\"))\n",
    "print('|' + '-' * 77 + '|')\n",
    "per_domain_acc = []\n",
    "for domain in np.unique(d).astype(int)[1:]:\n",
    "    Xt = X[np.where(d == domain)[0]]\n",
    "    yt = y[np.where(d == domain)[0]]\n",
    "    accs = cval_performance(wclf, X, y, d, domain, baseline=False)\n",
    "    per_domain_acc.append(np.mean(accs))\n",
    "    print(\"|{:^25}|{:^25}|{:^25}|\".format(domain,\n",
    "                                          np.round(np.mean(100 * accs), 3),\n",
    "                                          np.round(100 * np.std(accs), 3)))\n",
    "print(\"|{:^25}|{:^25}|{:^25}|\".format('Score',\n",
    "                                      np.round(100 * np.mean(per_domain_acc), 3),\n",
    "                                      np.round(100 * np.std(per_domain_acc), 3)))\n",
    "print('-' * 79)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3 id=\"KMM\">Kernel Mean Matching</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "K_grid = ['linear', 'rbf']\n",
    "G_grid = [[1e-3], [1e-2], [1e-1], None]\n",
    "B_grid = [1, 1e+1, 1e+2, 1e+3]\n",
    "kmm_cval_result = {}\n",
    "\n",
    "print('-' * (21 * 5) + '-')\n",
    "print('|{:^20}|{:^20}|{:^20}|{:^20}|{:^20}|'.format('C', 'Kernel', 'G', 'B', 'Score'))\n",
    "print('-' * (21 * 5) + '-')\n",
    "for C in [1, 15, 25]:\n",
    "    clf = SVC(C=C, kernel='linear')\n",
    "    for K in K_grid:\n",
    "        for G in G_grid:\n",
    "            for B in B_grid:\n",
    "                if K == 'linear' and G is not None:\n",
    "                    pass\n",
    "                else:\n",
    "                    wclf = WeightedClassifier(clf, KMM, kernel=K, kernel_param=G, B=B)\n",
    "                    per_domain_acc = []\n",
    "                    for domain in np.unique(d).astype(int)[1:]:\n",
    "                        Xt = X[np.where(d == domain)[0]]\n",
    "                        yt = y[np.where(d == domain)[0]]\n",
    "                        accs = cval_performance(wclf, Xs, ys, Xt, yt, n_folds=5, baseline=False)\n",
    "                        per_domain_acc.append(np.mean(accs))\n",
    "                    kmm_cval_result['{},{},{},{}'.format(C, K, G, B)] = (np.mean(per_domain_acc),\n",
    "                                                                         np.std(per_domain_acc))\n",
    "                    print('|{:^20}|{:^20}|{:^20}|{:^20}|{:^20}|'.format(C, K, str(G), B,\n",
    "                                                                        np.mean(per_domain_acc)))\n",
    "                    \n",
    "pprint.pprint(kmm_cval_result, width=1)\n",
    "with open('./results/KMM_cval.json', 'w') as f:\n",
    "    f.write(json.dumps(kmm_cval_result))\n",
    "    \n",
    "    \n",
    "clf = SVC(C=15, kernel='linear')\n",
    "wclf = WeightedClassifier(clf, KMM, kernel='rbf', kernel_param=[1e-3], B=1000)\n",
    "print('-' * 79)\n",
    "print('|{:^77}|'.format('Baseline'))\n",
    "print('-' * 79)\n",
    "print(\"|{:^25}|{:^25}|{:^25}|\".format(\"Domain\", \"Mean Accuracy\", \"Std Deviation\"))\n",
    "print('-' * 77)\n",
    "acc_per_domain = []\n",
    "for domain in np.unique(d).astype(int)[1:]:\n",
    "    Xt = X[np.where(d == domain)[0]]\n",
    "    yt = y[np.where(d == domain)[0]]\n",
    "    accs = cval_performance(wclf, X, y, d, domain, baseline=False, normalization=\"normalization\")\n",
    "    print(\"|{:^25}|{:^25}|{:^25}|\".format(domain,\n",
    "                                          np.round(100 * np.mean(accs), 3),\n",
    "                                          np.round(100 * np.std(accs), 3)))\n",
    "    acc_per_domain.append(np.mean(accs))\n",
    "print(\"|{:^25}|{:^25}|{:^25}|\".format('Score',\n",
    "                                      np.round(100 * np.mean(acc_per_domain), 3),\n",
    "                                      np.round(100 * np.std(acc_per_domain), 3)))\n",
    "print('-' * 79)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.7 64-bit ('OptimalTransport': conda)",
   "language": "python",
   "name": "python37764bitoptimaltransportconda38cd42aab3c640dc8d34473d4e53c23f"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}