wilcoxon_stats.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "interested-ebony",
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.statistical_test import wilcoxon_statistical_test\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import json\n",
    "\n",
    "def search_vs_nces_stat_tests(kb_emb_model='ConEx'):\n",
    "    print(\"### Semantic Bible ###\\n\")\n",
    "    with open(f'datasets/semantic_bible/Results/NCES2_{kb_emb_model}_Ensemble.json') as file:\n",
    "        nces = json.load(file)\n",
    "    f1_nces = nces['SetTransformer_I32+SetTransformer_I64']['f1']['values']\n",
    "    acc_nces = nces['SetTransformer_I32+SetTransformer_I64']['acc']['values']\n",
    "    time_nces = nces['SetTransformer_I32+SetTransformer_I64']['time']['values']\n",
    "    with open('datasets/semantic_bible/Results/CELOE_results.json') as celoe_file:\n",
    "        celoe = json.load(celoe_file)\n",
    "    with open('datasets/semantic_bible/Results/EvoLearner_results.json') as evo_file:\n",
    "        evo = json.load(evo_file)\n",
    "    f1_celoe = celoe['F-measure']\n",
    "    acc_evo = evo['Accuracy']\n",
    "    time_celoe = celoe['Runtime']\n",
    "    _, p1 = wilcoxon_statistical_test(f1_nces, f1_celoe)\n",
    "    _, p2 = wilcoxon_statistical_test(time_nces, time_celoe)\n",
    "    _, p3 = wilcoxon_statistical_test(acc_nces, acc_evo)\n",
    "    print(\"\\n\")\n",
    "    for metric, p in zip(['F1', 'RunTime', 'Accuracy'], [p1,p2,p3]):\n",
    "        print(f'On {metric} of Semantic Bible KG, p_value = ', p)\n",
    "        if p<=0.05:\n",
    "            print('* Probably different distributions *')\n",
    "        else:\n",
    "            print('Probably the same distribution')\n",
    "        print()\n",
    "        \n",
    "    print(\"### Mutagenesis ###\\n\")\n",
    "    with open(f'datasets/mutagenesis/Results/NCES2_{kb_emb_model}_Ensemble.json') as file:\n",
    "        nces = json.load(file)\n",
    "    f1_nces = nces['SetTransformer_I32+SetTransformer_I128']['f1']['values']\n",
    "    acc_nces = nces['SetTransformer_I32+SetTransformer_I128']['acc']['values']\n",
    "    time_nces = nces['SetTransformer_I32+SetTransformer_I128']['time']['values']\n",
    "    with open('datasets/mutagenesis/Results/EvoLearner_results.json') as evo_file:\n",
    "        evo = json.load(evo_file)\n",
    "    f1_evo = evo['F-measure']\n",
    "    acc_evo = evo['Accuracy']\n",
    "    time_evo = evo['Runtime']\n",
    "    _, p1 = wilcoxon_statistical_test(f1_nces, f1_evo)\n",
    "    _, p2 = wilcoxon_statistical_test(time_nces, time_evo)\n",
    "    _, p3 = wilcoxon_statistical_test(acc_nces, acc_evo)\n",
    "    print(\"\\n\")\n",
    "    for metric, p in zip(['F1', 'RunTime', 'Accuracy'], [p1,p2,p3]):\n",
    "        print(f'On {metric} of Mutagenesis KG, p_value = ', p)\n",
    "        if p<=0.05:\n",
    "            print('* Probably different distributions *')\n",
    "        else:\n",
    "            print('Probably the same distribution')\n",
    "        print()\n",
    "        \n",
    "    print(\"### Carcinogenesis ###\\n\")    \n",
    "    with open(f'datasets/carcinogenesis/Results/NCES2_{kb_emb_model}_Ensemble.json') as file:\n",
    "        nces = json.load(file)\n",
    "    f1_nces = nces['SetTransformer_I64+SetTransformer_I128']['f1']['values']\n",
    "    acc_nces = nces['SetTransformer_I64+SetTransformer_I128']['acc']['values']\n",
    "    time_nces = nces['SetTransformer_I64+SetTransformer_I128']['time']['values']\n",
    "    with open('datasets/carcinogenesis/Results/EvoLearner_results.json') as evo_file:\n",
    "        evo = json.load(evo_file)\n",
    "    f1_evo = evo['F-measure']\n",
    "    acc_evo = evo['Accuracy']\n",
    "    time_evo = evo['Runtime']\n",
    "    _, p1 = wilcoxon_statistical_test(f1_nces, f1_evo)\n",
    "    _, p2 = wilcoxon_statistical_test(time_nces, time_evo)\n",
    "    _, p3 = wilcoxon_statistical_test(acc_nces, acc_evo)\n",
    "    print(\"\\n\")\n",
    "    for metric, p in zip(['F1', 'RunTime', 'Accuracy'], [p1,p2,p3]):\n",
    "        print(f'On {metric} of Carcinogenesis KG, p_value = ', p)\n",
    "        if p<=0.05:\n",
    "            print('* Probably different distributions *')\n",
    "        else:\n",
    "            print('Probably the same distribution')\n",
    "        print()\n",
    "        \n",
    "    print(\"### Vicodi ###\\n\")    \n",
    "    with open(f'datasets/vicodi/Results/NCES2_{kb_emb_model}_Ensemble.json') as file:\n",
    "        nces = json.load(file)\n",
    "    f1_nces = nces['SetTransformer_I32+SetTransformer_I64+SetTransformer_I128']['f1']['values']\n",
    "    acc_nces = nces['SetTransformer_I32+SetTransformer_I64+SetTransformer_I128']['acc']['values']\n",
    "    time_nces = nces['SetTransformer_I32+SetTransformer_I64+SetTransformer_I128']['time']['values']\n",
    "    with open('datasets/vicodi/Results/EvoLearner_results.json') as evo_file:\n",
    "        evo = json.load(evo_file)\n",
    "    f1_evo = evo['F-measure']\n",
    "    time_evo = evo['Runtime']\n",
    "    _, p1 = wilcoxon_statistical_test(f1_nces, f1_evo)\n",
    "    _, p2 = wilcoxon_statistical_test(time_nces, time_evo)\n",
    "    _, p3 = wilcoxon_statistical_test(acc_nces, acc_evo)\n",
    "    for metric, p in zip(['F1', 'RunTime', 'Accuracy'], [p1,p2,p3]):\n",
    "        print(f'On {metric} of Vicodi KG, p_value = ', p)\n",
    "        if p<=0.05:\n",
    "            print('* Probably different distributions *')\n",
    "        else:\n",
    "            print('Probably the same distribution')\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "addressed-catch",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "### Semantic Bible ###\n",
      "\n",
      "stat=1072.000, p=0.043\n",
      "stat=0.000, p=0.000\n",
      "stat=1692.500, p=0.113\n",
      "\n",
      "\n",
      "On F1 of Semantic Bible KG, p_value =  0.04293467862219775\n",
      "* Probably different distributions *\n",
      "\n",
      "On RunTime of Semantic Bible KG, p_value =  3.896559845095909e-18\n",
      "* Probably different distributions *\n",
      "\n",
      "On Accuracy of Semantic Bible KG, p_value =  0.11289968058899645\n",
      "Probably the same distribution\n",
      "\n",
      "### Mutagenesis ###\n",
      "\n",
      "stat=2003.000, p=0.387\n",
      "stat=0.000, p=0.000\n",
      "stat=1345.500, p=0.001\n",
      "\n",
      "\n",
      "On F1 of Mutagenesis KG, p_value =  0.3867988925025292\n",
      "Probably the same distribution\n",
      "\n",
      "On RunTime of Mutagenesis KG, p_value =  3.896559845095909e-18\n",
      "* Probably different distributions *\n",
      "\n",
      "On Accuracy of Mutagenesis KG, p_value =  0.0012865604884367975\n",
      "* Probably different distributions *\n",
      "\n",
      "### Carcinogenesis ###\n",
      "\n",
      "stat=895.000, p=0.000\n",
      "stat=0.000, p=0.000\n",
      "stat=1160.000, p=0.000\n",
      "\n",
      "\n",
      "On F1 of Carcinogenesis KG, p_value =  2.1203785238395784e-06\n",
      "* Probably different distributions *\n",
      "\n",
      "On RunTime of Carcinogenesis KG, p_value =  3.896559845095909e-18\n",
      "* Probably different distributions *\n",
      "\n",
      "On Accuracy of Carcinogenesis KG, p_value =  0.00022113800397504876\n",
      "* Probably different distributions *\n",
      "\n",
      "### Vicodi ###\n",
      "\n",
      "stat=730.000, p=0.000\n",
      "stat=0.000, p=0.000\n",
      "stat=1528.500, p=0.002\n",
      "On F1 of Vicodi KG, p_value =  1.1501740433179402e-07\n",
      "* Probably different distributions *\n",
      "\n",
      "On RunTime of Vicodi KG, p_value =  3.896559845095909e-18\n",
      "* Probably different distributions *\n",
      "\n",
      "On Accuracy of Vicodi KG, p_value =  0.002276729277049842\n",
      "* Probably different distributions *\n",
      "\n"
     ]
    }
   ],
   "source": [
    "search_vs_nces_stat_tests()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "supreme-mobile",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "### Semantic Bible ###\n",
      "\n",
      "stat=904.500, p=0.006\n",
      "stat=0.000, p=0.000\n",
      "stat=1344.000, p=0.034\n",
      "\n",
      "\n",
      "On F1 of Semantic Bible KG, p_value =  0.005986231457780366\n",
      "* Probably different distributions *\n",
      "\n",
      "On RunTime of Semantic Bible KG, p_value =  3.896559845095909e-18\n",
      "* Probably different distributions *\n",
      "\n",
      "On Accuracy of Semantic Bible KG, p_value =  0.034103629082319856\n",
      "* Probably different distributions *\n",
      "\n",
      "### Mutagenesis ###\n",
      "\n",
      "stat=1572.000, p=0.078\n",
      "stat=0.000, p=0.000\n",
      "stat=1131.500, p=0.000\n",
      "\n",
      "\n",
      "On F1 of Mutagenesis KG, p_value =  0.07818518625620051\n",
      "Probably the same distribution\n",
      "\n",
      "On RunTime of Mutagenesis KG, p_value =  3.896559845095909e-18\n",
      "* Probably different distributions *\n",
      "\n",
      "On Accuracy of Mutagenesis KG, p_value =  0.00036572784223708486\n",
      "* Probably different distributions *\n",
      "\n",
      "### Carcinogenesis ###\n",
      "\n",
      "stat=961.000, p=0.000\n",
      "stat=0.000, p=0.000\n",
      "stat=1110.500, p=0.000\n",
      "\n",
      "\n",
      "On F1 of Carcinogenesis KG, p_value =  2.0339936014109157e-05\n",
      "* Probably different distributions *\n",
      "\n",
      "On RunTime of Carcinogenesis KG, p_value =  3.896559845095909e-18\n",
      "* Probably different distributions *\n",
      "\n",
      "On Accuracy of Carcinogenesis KG, p_value =  0.0002617819957684687\n",
      "* Probably different distributions *\n",
      "\n",
      "### Vicodi ###\n",
      "\n",
      "stat=784.000, p=0.000\n",
      "stat=0.000, p=0.000\n",
      "stat=1457.000, p=0.001\n",
      "On F1 of Vicodi KG, p_value =  3.696426115434257e-07\n",
      "* Probably different distributions *\n",
      "\n",
      "On RunTime of Vicodi KG, p_value =  3.896559845095909e-18\n",
      "* Probably different distributions *\n",
      "\n",
      "On Accuracy of Vicodi KG, p_value =  0.0009365066833412846\n",
      "* Probably different distributions *\n",
      "\n"
     ]
    }
   ],
   "source": [
    "search_vs_nces_stat_tests('Distmult')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "focused-manual",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nces",
   "language": "python",
   "name": "nces"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}