Ch7Sct4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "using Gadfly\n",
    "using Colors\n",
    "using DataFrames\n",
    "using Distributed\n",
    "addprocs(...); # set number depending on number of cores available on your computer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@everywhere begin\n",
    "    using Distributions\n",
    "    using StatsBase\n",
    "    using Distances\n",
    "    using LinearAlgebra\n",
    "    using SharedArrays\n",
    "    using JuliennedArrays\n",
    "    using Random\n",
    "    using Bootstrap\n",
    "    using DelimitedFiles\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@everywhere begin\n",
    "    const numb_hyp = 11\n",
    "    const numb_agent = 50\n",
    "    const numb_toss = 100\n",
    "    const likelihood_heads = range(0, stop=1, length=numb_hyp)\n",
    "    const likelihood_tails = range(1, stop=0, length=numb_hyp)\n",
    "    const numb_sim = 50\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "function gen_brew_colors(n) # to create your own colors, here based on one of the brewer series\n",
    "    cs = distinguishable_colors(n, \n",
    "        [colorant\"#66c2a5\", colorant\"#fc8d62\", colorant\"#8da0cb\", colorant\"#e78ac3\",\n",
    "            colorant\"#a6d854\", colorant\"#ffd92f\", colorant\"#e5c494\", colorant\"#b3b3b3\"],\n",
    "        lchoices=Float64[58, 45, 72.5, 90],\n",
    "        transform=c->deuteranopic(c, 0.1),\n",
    "        cchoices=Float64[20,40],\n",
    "        hchoices=[75,51,35,120,180,210,270,310]\n",
    "    )\n",
    "    convert(Vector{Color}, cs)\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data generation function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following function generates sequences of coin tosses, one per agent, where the bias of the coin is given by `bias` and the length of the sequences by `numb_toss`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@everywhere function generate_data(bias::Float64)\n",
    "    sim_dat = Array{Int64,2}(undef, numb_agent, numb_toss)\n",
    "    for i in 1:numb_agent\n",
    "        sim_dat[i, :] = rand(Bernoulli(bias), numb_toss)\n",
    "    end\n",
    "    return sim_dat\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Update rules"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These are the update rules to be compared. The first is Bayes' rule, which has `bonus` as a kind of dummy parameter, to make sure it has to right form for the evolutionary computing to be carried out later. The other rules can in fact also do duty as Bayes' rule (which is the limiting case of each of them, obtained by setting `bonus = 0`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Bayes' rule\n",
    "@everywhere function b_upd(probs::Array{Float64,1}, dat::Array{Int64,1}, toss_num::Int64, bonus::Float64)\n",
    "    if dat[toss_num] == 1\n",
    "        @. (probs * likelihood_heads) / $dot(probs, likelihood_heads)\n",
    "    else\n",
    "        @. (probs * likelihood_tails) / $dot(probs, likelihood_tails)\n",
    "    end\n",
    "end\n",
    "\n",
    "# EXPL\n",
    "@everywhere function expl_upd(probs::Array{Float64,1}, dat::Array{Int64,1}, toss_num::Int64, bonus::Float64)\n",
    "    val::Float64 = mean(dat[1:toss_num]) * (numb_hyp - 1.0) + 1.0\n",
    "    vec::Array{Float64,1} = if dat[toss_num] == 1\n",
    "        @. (probs * likelihood_heads) / $dot(probs, likelihood_heads)\n",
    "    else\n",
    "        @. (probs * likelihood_tails) / $dot(probs, likelihood_tails)\n",
    "    end\n",
    "    if val % 1 == .5\n",
    "        vec[floor(Int, val)] += bonus*0.5\n",
    "        vec[ceil(Int, val)] += bonus*0.5\n",
    "    else\n",
    "        vec[round(Int, val)] += bonus\n",
    "    end\n",
    "    return vec / (1.0 + bonus)\n",
    "end\n",
    "\n",
    "#Good's rule; with λ=2 (default value), we obtain the rule L2 from Douven & Schupbach, 2015 (Frontiers paper)\n",
    "@everywhere function good_bonus(probs::Array{Float64,1}, res::Int64, λ=2.0)\n",
    "    pE::Float64 = res == 1 ? dot(probs, likelihood_heads) : dot(probs, likelihood_tails)\n",
    "    gb::Array{Float64,1} = res == 1 ? log.(likelihood_heads ./ pE) : log.(likelihood_tails ./ pE)\n",
    "    function rsc(i)\n",
    "        if i >= 0\n",
    "            1 - exp(2λ^2 * -i^2)\n",
    "        else\n",
    "            -1 + exp(2λ^2 * -i^2)\n",
    "        end\n",
    "    end\n",
    "    return map(rsc, gb)\n",
    "end\n",
    "\n",
    "# γ is the proportion of the probability that gets added as a bonus, so necessarily γ ⩽ 1\n",
    "@everywhere function good_upd(probs::Array{Float64, 1}, dat::Array{Int64,1}, toss_num::Int64, γ::Float64)\n",
    "    res::Int64 = dat[toss_num]\n",
    "    probvec::Array{Float64,1} = if res == 1\n",
    "        @. (probs * likelihood_heads) / $dot(probs, likelihood_heads)\n",
    "    else\n",
    "        @. (probs * likelihood_tails) / $dot(probs, likelihood_tails)\n",
    "    end\n",
    "    goodvec::Array{Float64,1} = probvec + γ .* (probvec .* good_bonus(probs, res))\n",
    "    return goodvec / sum(goodvec)\n",
    "end\n",
    "\n",
    "\n",
    "# Popper's rule; in pop_upd, γ is the proportion of the probability that gets added as a bonus, so necessarily γ ⩽ 1\n",
    "@everywhere function pop_bonus(probs::Array{Float64,1}, res::Int64)\n",
    "    pE::Float64 = res == 1 ? dot(probs, likelihood_heads) : dot(probs, likelihood_tails)\n",
    "    pb::Array{Float64,1} = res == 1 ? (likelihood_heads .- pE) ./ (likelihood_heads .+ pE) : (likelihood_tails .- pE) ./ (likelihood_tails .+ pE)\n",
    "end\n",
    "\n",
    "@everywhere function pop_upd(probs::Array{Float64,1}, dat::Array{Int64,1}, toss_num::Int, γ::Float64)\n",
    "    res::Int64 = dat[toss_num]\n",
    "    probvec::Array{Float64,1} = if res == 1\n",
    "        @. (probs * likelihood_heads) / $dot(probs, likelihood_heads)\n",
    "    else\n",
    "        @. (probs * likelihood_tails) / $dot(probs, likelihood_tails)\n",
    "    end\n",
    "    popvec::Array{Float64,1} = probvec + γ .* (probvec .* pop_bonus(probs, res))\n",
    "    return popvec / sum(popvec)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@everywhere function social_learning(rule::Function, dist::DataType, averaging, bonus::Float64, ϵ::Float64, sdat::Array{Int64,2})\n",
    "    UPD = Array{Float64,3}(undef, numb_hyp, numb_agent, numb_toss + 1)\n",
    "    UPD[:, :, 1] = repeat(fill(1/numb_hyp, numb_hyp), 1, numb_agent)\n",
    "    PROB = Array{Float64,2}(undef, numb_hyp, numb_agent)\n",
    "    f(x::Float64) = (x<=ϵ)::Bool\n",
    "    @inbounds for t in 1:numb_toss\n",
    "        for i in 1:numb_agent\n",
    "            PROB[:, i] = rule(UPD[:, i, t], sdat[i, :], t, bonus)\n",
    "        end\n",
    "        prob_dist::Array{Float64,2} = pairwise(dist(), PROB, dims=2)\n",
    "        peers::Array{Bool,2} = map(f, prob_dist)\n",
    "        if averaging == mean\n",
    "            @inbounds for i::Int in 1:numb_agent\n",
    "               v = @view PROB[:, peers[i, :]]\n",
    "               UPD[:, i, t + 1] = mean(v', dims=1)\n",
    "            end\n",
    "        else\n",
    "            @inbounds for i::Int in 1:numb_agent\n",
    "               q = map(averaging, Slices(PROB[:, peers[i, :]], False(), True()))\n",
    "               UPD[:, i, t + 1] = q / sum(q)\n",
    "            end\n",
    "        end\n",
    "    end\n",
    "    return UPD\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@everywhere function survWei(res::Tuple{Int64,Int64},\n",
    "                             hyp::Int64,\n",
    "                             a::Float64=rand(Uniform(1, 10)),\n",
    "                             b::Float64=rand(Uniform(1, 10)),\n",
    "                             shape::Float64=rand(Uniform(.5, 5)),\n",
    "                             scale::Float64=rand(Uniform(50, 250)))\n",
    "\n",
    "    p = first(res)\n",
    "\n",
    "    if last(res) == hyp\n",
    "        1 - (cdf(Weibull(shape, scale), p) / a) # probability goes down if right intervention is made (which is made when the truth is assigned a probability above thresh)\n",
    "    elseif last(res) == -1\n",
    "        1 - cdf(Weibull(shape, scale), numb_toss + 1) # if no intervention is made, output survival probability at last time step\n",
    "    else\n",
    "        (1 - cdf(Weibull(shape, scale), p)) / b # probability goes down if wrong intervention is made (which happens if a false hypothesis is assigned a probabilty above thresh)\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@everywhere function survGam(res::Tuple{Int64,Int64},\n",
    "                             hyp::Int64,\n",
    "                             a::Float64=rand(Uniform(1, 10)),\n",
    "                             b::Float64=rand(Uniform(1, 10)),\n",
    "                             shape::Float64=rand(Uniform(10, 16)),\n",
    "                             scale::Float64=rand(Uniform(10, 16)))\n",
    "\n",
    "    p = first(res)\n",
    "\n",
    "    if last(res) == hyp\n",
    "        1 - (cdf(Gamma(shape, scale), p) / a) # probability goes down if right intervention is made (which is made when the truth is assigned a probability above thresh)\n",
    "    elseif last(res) == -1\n",
    "        1 - cdf(Gamma(shape, scale), numb_toss + 1) # if no intervention is made, output survival probability at last time step\n",
    "    else\n",
    "        (1 - cdf(Gamma(shape, scale), p)) / b # probability goes down if wrong intervention is made (which happens if a false hypothesis is assigned a probabilty above thresh)\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@everywhere function treat_patients(rule::Function, dist::DataType, averaging, bonus::Float64, ϵ::Float64)\n",
    "    score = 0.0\n",
    "    i = 1\n",
    "    while i <= 100\n",
    "        rand_hyp::Int64 = rand(1:11)\n",
    "        dt = generate_data((rand_hyp - 1) / 10)\n",
    "        sl = social_learning(rule, dist, averaging, bonus, ϵ, dt)\n",
    "        m = mapslices(findmax, mapslices(x->x.>25, sum(mapslices(x->x.>.9, sl, dims=(1, 3)), dims=2), dims=1), dims=1)\n",
    "        v = reshape(m, 101)\n",
    "        c = [ first(v[i]) == true ? last(v[i]) : -1 for i in 1:length(v) ]\n",
    "        ff = something(findfirst(x->x.>0, c), 101)\n",
    "        cff = c[ff]\n",
    "        res = (ff, cff)\n",
    "        score += survWei(res, rand_hyp)\n",
    "        i += 1\n",
    "    end\n",
    "    return score / 100\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@everywhere function offspring(a::Int64, b::Int64, mat::Array{Any,2})\n",
    "    r::Int64 = rand(1:3)\n",
    "    s::Vector{Int64} = sample(1:3, r, replace=false, ordered=true)\n",
    "    sd::Vector{Int64} = setdiff(1:3, s)\n",
    "    c = Vector{Any}(undef, 5)\n",
    "    c[s] = mat[a, :][s]\n",
    "    c[sd] = mat[b, :][sd]\n",
    "    c[4] = c[1] == b_upd ? 0.0 : rand(truncated(Normal(mean([mat[a, 4], mat[b, 4]]), std(mat[:, 4])), 0.0, 1.0))\n",
    "    c[5] = rand(truncated(Normal(mean([mat[a, 5], mat[b, 5]]), std(mat[:, 5])), 0.0, 1.0))\n",
    "    return c::Vector{Any}\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@everywhere function new_generation(mat::Array{Any,2})\n",
    "    scrs = [ treat_patients(mat[i, :]...) for i in 1:36 ]\n",
    "    np = hcat(mat, scrs)\n",
    "    sel = np[sortperm(np[:, end], rev=true), :]\n",
    "    nnp = sel[1:18, 1:5]\n",
    "    smp1 = shuffle(1:18)\n",
    "    smp2 = shuffle(1:18)\n",
    "    offspr = Array{Any,2}(undef, 18, 5)\n",
    "    @inbounds for i in 1:18\n",
    "        offspr[i, :] = offspring(smp1[i], smp2[i], nnp)\n",
    "    end\n",
    "    return vcat(offspr, nnp), mean(scrs), std(scrs)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@everywhere function single_run(r)\n",
    "    rule_id = vcat(repeat([b_upd], 9), repeat([expl_upd], 9), repeat([good_upd], 9), repeat([pop_upd], 9))\n",
    "    c_vals = vcat(repeat([0.0], 9), rand(Uniform(), 27)) # bonus values are randomly chosen from [0, 1]\n",
    "    eps_vals = rand(Uniform(), 36) # same for ϵ\n",
    "    dist_id = repeat(vcat(repeat([Euclidean], 3), repeat([Cityblock], 3), repeat([KLDivergence], 3)), 4)\n",
    "    av_id = repeat([mean, geomean, harmmean], 12)\n",
    "    pop_start = hcat(rule_id, dist_id, av_id, c_vals, eps_vals)\n",
    "\n",
    "    run(`mkdir gen_data_new$r`)\n",
    "    open(\"gen_data_new$r/gen1.txt\", \"w\") do io\n",
    "        writedlm(io, pop_start)\n",
    "    end\n",
    "\n",
    "    scr = Array{Float64,1}(undef, 49)\n",
    "    scr_std = Array{Float64,1}(undef, 49)\n",
    "\n",
    "    old = pop_start\n",
    "    for i in 2:50\n",
    "        ng = new_generation(old)\n",
    "        new = ng[1]\n",
    "        scr[i - 1] = ng[2]\n",
    "        scr_std[i - 1] = ng[3]\n",
    "        open(\"gen_data_new$r/gen$i.txt\", \"w\") do io\n",
    "            writedlm(io, new)\n",
    "        end\n",
    "        old .= new\n",
    "        new = nothing\n",
    "        GC.gc()\n",
    "    end\n",
    "    open(\"gen_data_new$r/SCR.txt\", \"w\") do io\n",
    "        writedlm(io, scr)\n",
    "    end\n",
    "    open(\"gen_data_new$r/SCRstd.txt\", \"w\") do io\n",
    "        writedlm(io, scr_std)\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We run the function 15 times, in parallel."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pmap(single_run, 1:15);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load the data concerning the final generation in each run."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_gens = Array{Any,3}(undef, 36, 5, 15)\n",
    "\n",
    "for i in 1:15\n",
    "    final_gens[:, :, i] = readdlm(\"gen_data_new$i/gen50.txt\")\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Count of update rules represented in last generations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "countmap(final_gens[:, 1, :])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = plot(x = [\"Bayes\", \"EXPL\", \"Good\", \"Popper\"], y = [27, 88, 0, 425], Geom.bar,\n",
    "    Guide.xlabel(\"Rule\"),\n",
    "    Guide.ylabel(\"Count\"),\n",
    "    Scale.x_discrete,\n",
    "    style(bar_spacing=35pt))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Same for distance measures and, respectively, pooling methods."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dns = vcat([final_gens[:, :, i] for i in 1:15]...);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "expl_bns = dns[dns[:, 1] .== \"expl_upd\", 4]\n",
    "#good_bns = dns[dns[:, 1] .== \"good_upd\", 4] good's rule not present in final generations\n",
    "pop_bns = dns[dns[:, 1] .== \"pop_upd\", 4];"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = DataFrame(Rule = vcat(fill(\"EXPL\", length(expl_bns)), fill(\"Popper\", length(pop_bns))), C_value = vcat(expl_bns, pop_bns));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = plot(df, x=:C_value, color=:Rule, Geom.density(bandwidth=.02),\n",
    "     Coord.cartesian(xmin=-.0075, xmax=1.1),\n",
    "     Scale.color_discrete_manual(hcat(gen_brew_colors(4)[2:4][1], gen_brew_colors(4)[2:4][3])...),\n",
    "     Guide.xlabel(\"Bonus value\"),\n",
    "     Guide.ylabel(\"Density\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "countmap(final_gens[:, 2, :])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "countmap(final_gens[:, 3, :])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Mean and standard deviation of explanation bonus of best groups in last generations. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean(final_gens[1, 4, :][:]), std(final_gens[1, 4, :][:])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Same for $\\epsilon$ values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean(final_gens[1, 5, :][:]), std(final_gens[1, 5, :][:]) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Counts of types of update rule, types of metric, and types of pooling method -- to create figures."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_gens = Array{Any,3}(undef, 36, 5, 50)\n",
    "\n",
    "for i in 1:50\n",
    "    all_gens[:, :, i] = readdlm(\"gen_data_new3/gen$i.txt\")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ks = [keys(countmap(all_gens[:, 1, i])) for i in 1:50]\n",
    "vls = [values(countmap(all_gens[:, 1, i])) for i in 1:50]\n",
    "\n",
    "bayes = Vector{Int64}(undef, 50)\n",
    "good = Vector{Int64}(undef, 50)\n",
    "expl = Vector{Int64}(undef, 50)\n",
    "pop = Vector{Int64}(undef, 50)\n",
    "\n",
    "for i in 1:50\n",
    "    bayes[i] = Int(collect(vls[i])[findall(x->x==\"b_upd\", collect(ks[i]))] != [] && collect(vls[i])[findall(x->x==\"b_upd\", collect(ks[i]))][1])\n",
    "    good[i] = Int(collect(vls[i])[findall(x->x==\"good_upd\", collect(ks[i]))] != [] && collect(vls[i])[findall(x->x==\"good_upd\", collect(ks[i]))][1])\n",
    "    expl[i] = Int(collect(vls[i])[findall(x->x==\"expl_upd\", collect(ks[i]))] != [] && collect(vls[i])[findall(x->x==\"expl_upd\", collect(ks[i]))][1])\n",
    "    pop[i] = Int(collect(vls[i])[findall(x->x==\"pop_upd\", collect(ks[i]))] != [] && collect(vls[i])[findall(x->x==\"pop_upd\", collect(ks[i]))][1])\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rule_types = hcat(bayes, expl, good, pop);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "open(\"rule_types1.txt\", \"w\") do io\n",
    "    writedlm(io, rule_types)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ks = [keys(countmap(all_gens[:, 2, i])) for i in 1:50]\n",
    "vls = [values(countmap(all_gens[:, 2, i])) for i in 1:50]\n",
    "\n",
    "eucl = Vector{Int64}(undef, 50)\n",
    "city = Vector{Int64}(undef, 50)\n",
    "kl = Vector{Int64}(undef, 50)\n",
    "\n",
    "for i in 1:50\n",
    "    eucl[i] = Int(collect(vls[i])[findall(x->x==\"Euclidean\", collect(ks[i]))] != [] && collect(vls[i])[findall(x->x==\"Euclidean\", collect(ks[i]))][1])\n",
    "    city[i] = Int(collect(vls[i])[findall(x->x==\"Cityblock\", collect(ks[i]))] != [] && collect(vls[i])[findall(x->x==\"Cityblock\", collect(ks[i]))][1])\n",
    "    kl[i] = Int(collect(vls[i])[findall(x->x==\"KLDivergence\", collect(ks[i]))] != [] && collect(vls[i])[findall(x->x==\"KLDivergence\", collect(ks[i]))][1])\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dist_types = hcat(eucl, kl, city);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "open(\"dist_types1.txt\", \"w\") do io\n",
    "    writedlm(io, dist_types)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ks = [keys(countmap(all_gens[:, 3, i])) for i in 1:50]\n",
    "vls = [values(countmap(all_gens[:, 3, i])) for i in 1:50]\n",
    "\n",
    "mn = Vector{Int64}(undef, 50)\n",
    "gmn = Vector{Int64}(undef, 50)\n",
    "hmn = Vector{Int64}(undef, 50)\n",
    "\n",
    "for i in 1:50\n",
    "    mn[i] = Int(collect(vls[i])[findall(x->x==\"mean\", collect(ks[i]))] != [] && collect(vls[i])[findall(x->x==\"mean\", collect(ks[i]))][1])\n",
    "    gmn[i] = Int(collect(vls[i])[findall(x->x==\"geomean\", collect(ks[i]))] != [] && collect(vls[i])[findall(x->x==\"geomean\", collect(ks[i]))][1])\n",
    "    hmn[i] = Int(collect(vls[i])[findall(x->x==\"harmmean\", collect(ks[i]))] != [] && collect(vls[i])[findall(x->x==\"harmmean\", collect(ks[i]))][1])\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "av_types = hcat(mn, gmn, hmn);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "open(\"av_types1.txt\", \"w\") do io\n",
    "    writedlm(io, av_types)\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Mean and standard deviation of bonus values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c_mn = Float64[]\n",
    "c_std = Float64[]\n",
    "\n",
    "for i in 1:50\n",
    "    lst = all_gens[:, :, i]\n",
    "    m, s = mean_and_std(convert(Array{Float64,1}, lst[:, 4]))\n",
    "    push!(c_mn, m)\n",
    "    push!(c_std, s)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bonus_gen = hcat(round.(c_mn, digits=6), round.(c_std, digits=6));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "open(\"gen_bonus1.txt\", \"w\") do io\n",
    "    writedlm(io, bonus_gen)\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Same for ϵ values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eps_mn = Float64[]\n",
    "eps_std = Float64[]\n",
    "\n",
    "for i in 1:50\n",
    "    lst = all_gens[:, :, i]\n",
    "    m, s = mean_and_std(convert(Array{Float64,1}, lst[:, 5]))\n",
    "    push!(eps_mn, m)\n",
    "    push!(eps_std, s)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eps_gen = hcat(round.(eps_mn, digits=6), round.(eps_std, digits=6));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "open(\"eps1.txt\", \"w\") do io\n",
    "    writedlm(io, eps_gen)\n",
    "end"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.5.3",
   "language": "julia",
   "name": "julia-1.5"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.3.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}