run_lm2seq_end2end.py

# coding: utf-8
from src.train_and_evaluate import *
from src.models import *
import time
import torch.optim
from src.load_data import *
from src.num_transfer import *
from src.expression_tree import *
from src.log_utils import *
from src.calculation import *
# from src.expressions_transfer import *
from src.data_utils import get_pretrained_embedding_weight, pad_seq
from transformers import BertTokenizer, AdamW
import argparse
from itertools import groupby

torch.cuda.set_device(1)
# USE_CUDA = torch.cuda.is_available()
# batch_size = 16
# grad_acc_steps = 8  # 使用grad_acc_steps步来完成batch_size的训练，每一步：batch_size // grad_acc_steps
# embedding_size = 128
# hidden_size = 768
# n_epochs = 80
# bert_learning_rate = 5e-5
# bert_path = "./pretrained_lm/chinese-bert-wwm"
# learning_rate = 1e-3
# weight_decay = 2e-5
# beam_size = 5
# use_teacher_forcing = 1.0
# gclip = 0
# beam_search = True
# fold_num = 5
# n_layers = 1
# drop_out = 0.5
# random_seed = 1
# var_nums = []
# dataset_name = "mawps"
# ckpt_dir = "Math23K"
# data_path = "../data/Math_23K.json"

parser = argparse.ArgumentParser()
parser.add_argument('--random_seed', type=int, default=1)
parser.add_argument('--batch_size', type=int, default=16)
parser.add_argument('--grad_acc_steps', type=int, default=8)
parser.add_argument('--embedding_size', type=int, default=128)
parser.add_argument('--hidden_size', type=int, default=768)
parser.add_argument('--n_epochs', type=int, default=80)
parser.add_argument('--stage1_learning_rate', type=float, default=1e-5)
parser.add_argument('--bert_learning_rate', type=float, default=5e-5)
parser.add_argument('--bert_path', type=str, default="./pretrained_lm/chinese-bert-wwm")
parser.add_argument('--learning_rate', type=float, default=1e-3)
parser.add_argument('--stage1_weight_decay', type=float, default=5e-6)
parser.add_argument('--weight_decay', type=float, default=2e-5)

parser.add_argument('--enable_beam_search', action='store_true')
parser.add_argument('--beam_size', type=int, default=5)
parser.add_argument('--fold_num', type=int, default=5)

parser.add_argument('--n_layers', type=int, default=1)
parser.add_argument('--drop_out', type=float, default=0.5)
parser.add_argument('--use_teacher_forcing', type=float, default=1.0)
parser.add_argument('--use_clip', action='store_true')
parser.add_argument('--gclip', type=float, default=0.0)
parser.add_argument('--dataset_name', type=str, default='Math23K')
args = parser.parse_args()


USE_CUDA = torch.cuda.is_available()
batch_size = args.batch_size  # 16
grad_acc_steps = args.grad_acc_steps  # 8  # 使用grad_acc_steps步来完成batch_size的训练，每一步：batch_size // grad_acc_steps
embedding_size = args.embedding_size  # 128
hidden_size = args.hidden_size  # 768
n_epochs = args.n_epochs  # 80
stage1_learning_rate = args.stage1_learning_rate  # 1e-5
bert_learning_rate = args.bert_learning_rate  # 5e-5
bert_path = args.bert_path  # "./pretrained_lm/chinese-bert-wwm"
learning_rate = args.learning_rate  # 1e-3
weight_decay = args.weight_decay  # 2e-5
stage1_weight_decay = args.stage1_weight_decay  # 5e-6
beam_size = args.beam_size  # 5
use_teacher_forcing = args.use_teacher_forcing  # 1.0
use_clip = args.use_clip
gclip = args.gclip  # 0
beam_search = args.enable_beam_search  # True
fold_num = args.fold_num  # 5
n_layers = args.n_layers  # 1
drop_out = args.drop_out  # 0.5
random_seed = args.random_seed  # 1
var_nums = []
dataset_name = args.dataset_name  # "mawps"
ckpt_dir = "Math23K"
data_path = "../data/Math_23K.json"
num_labels = 5 #category class number
use_sentence_index = True
aggregate_mode = 'mean' #'cls', 'mean', 'sep'
rnn_type = 'transformer'

if dataset_name == "Math23K":
    var_nums = []
    ckpt_dir = "Math23K_b2s"
    bert_path = "./pretrained_lm/chinese-bert-wwm"
    data_dir = "./dataset/math23k/"
    data_path = data_dir + "Math_23K.json"
elif dataset_name == "mawps":
    var_nums = []
    ckpt_dir = "mawps_b2s"
    bert_path = "./pretrained_lm/bert-base-uncased"  # 需要修改为英文的预训练
    data_dir = "./dataset/mawps/"
    data_path = data_dir + "mawps_combine.json"
elif dataset_name == "ALG514":
    var_nums = ['x', 'y']
    ckpt_dir = "ALG514_b2s_new_retrain5"
    bert_path = "bert-base-uncased"  # 需要修改为英文的预训练
    data_path = "./dataset/alg514/questions_normalization_v2.json"
    stage1_path = "./benchmark_labels/label_v3_withQ.json"
elif dataset_name == "hmwp":
    var_nums = ['x', 'y']
    ckpt_dir = "hmwp_b2s"
    bert_path = "./pretrained_lm/chinese-bert-wwm"
    data_path = "./dataset/hmwp/hmwp.json"
elif dataset_name == "cm17k":
    var_nums = ['x', 'y']
    ckpt_dir = "cm17k_b2s"
    bert_path = "./pretrained_lm/chinese-bert-wwm"
    data_path = "./dataset/cm17k/questions.json"

ckpt_dir = ckpt_dir + '_' + str(n_epochs) + '_' + str(batch_size) + '_' + str(embedding_size) + '_' + str(hidden_size) + \
           'stage1_blr' + str(stage1_learning_rate) + 'stage1_wd' + str(stage1_weight_decay) + \
           '_blr' + str(bert_learning_rate) + '_lr' + str(learning_rate) + '_wd' + str(weight_decay) + '_do' + str(drop_out)
if beam_search:
    ckpt_dir = ckpt_dir + '_' + 'beam_search' + str(beam_size)


save_dir = os.path.join("./models", ckpt_dir)
if not os.path.exists(save_dir):
    os.mkdir(save_dir)

log_file = os.path.join(save_dir, 'log')
create_logs(log_file)

for fold_id in range(fold_num):
    if not os.path.exists(os.path.join(save_dir, 'fold-'+str(fold_id))):
        os.mkdir(os.path.join(save_dir, 'fold-'+str(fold_id)))

# data = load_math23k_data("../dataset/math23k/Math_23K.json")
# data = load_math23k_data(data_path)
# pairs, generate_nums, copy_nums = transfer_math23k_num(data)
pairs = None
generate_nums = None
copy_nums = None
if dataset_name == "Math23K":
    data = load_math23k_data(data_path)
    pairs, generate_nums, copy_nums = transfer_math23k_num(data)
elif dataset_name == "mawps":
    data = load_mawps_data(data_path)
    pairs, generate_nums, copy_nums = transfer_mawps_num(data)
elif dataset_name == "ALG514":
    #data = load_alg514_data(data_path)
    data = load_alg514_data(data_path, stage1_path)
    pairs, generate_nums, copy_nums = transfer_alg514_num(data)
elif dataset_name == "hmwp":
    data = load_hmwp_data(data_path)
    pairs, generate_nums, copy_nums = transfer_hmwp_num(data)
elif dataset_name == "cm17k":
    data = load_cm17k_data(data_path)
    pairs, generate_nums, copy_nums = transfer_cm17k_num(data)

temp_pairs = []
for p in pairs:
    # ept = ExpressionTree()
    # ept.build_tree_from_infix_expression(p["out_seq"])
    # p['out_seq'] = ept.get_prefix_expression()
    temp_pairs.append(p)
pairs = temp_pairs

fold_size = int(len(pairs) / fold_num)
fold_pairs = []
for split_fold in range((fold_num - 1)):
    fold_start = fold_size * split_fold
    fold_end = fold_size * (split_fold + 1)
    fold_pairs.append(pairs[fold_start:fold_end])
fold_pairs.append(pairs[(fold_size * (fold_num - 1)):])

last_acc_fold = []
best_val_acc_fold = []
#last_train_acc_fold = []
#best_train_acc_fold = []
all_acc_data = []
#all_acc_data_train = []

for fold in range(fold_num):
    pairs_tested = []
    pairs_trained = []
    for fold_t in range(fold_num):
        if fold_t == fold:
            pairs_tested += fold_pairs[fold_t]
        else:
            pairs_trained += fold_pairs[fold_t]

    random.seed(random_seed)
    import numpy as np
    np.random.seed(random_seed)
    torch.manual_seed(random_seed)  # cpu
    if USE_CUDA:
        torch.cuda.manual_seed(random_seed) # gpu
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False

    bert_tokenizer = BertTokenizer.from_pretrained(bert_path)
    bert_tokenizer.add_tokens(['[NUM]'])

    _, output_lang, train_pairs, test_pairs = prepare_data(pairs_trained, pairs_tested, 5, generate_nums,
                                                        copy_nums, tree=False, use_lm=True,
                                                        use_group_num=False)

    embedding_weight = None

    # Initialize models
    #encoder = PLMEncoderSeq(model_path=bert_path)
    stage1_encoder = Stage1_Encoder(num_labels, bert_tokenizer, use_sentence_index, aggregate_mode, rnn_type)
    encoder = PLMEncoderSeq_v2(model_path=bert_path)
    decoder = AttnDecoderRNN(hidden_size=hidden_size, embedding_size=embedding_size,
                             input_size=output_lang.n_words, output_size=output_lang.n_words,
                             n_layers=n_layers, dropout=drop_out)
    # the embedding layer is  only for generated number embeddings, operators, and paddings
    stage1_encoder_optimizer = AdamW(stage1_encoder.parameters(), lr=stage1_learning_rate, weight_decay=stage1_weight_decay)
    encoder_optimizer = AdamW(encoder.parameters(), lr=bert_learning_rate, weight_decay=weight_decay)
    decoder_optimizer = torch.optim.Adam(decoder.parameters(), lr=learning_rate, weight_decay=weight_decay)

    stage1_encoder_scheduler = torch.optim.lr_scheduler.StepLR(stage1_encoder_optimizer, step_size=max(n_epochs//4,1), gamma=0.5)
    encoder_scheduler = torch.optim.lr_scheduler.StepLR(encoder_optimizer, step_size=max(n_epochs//4,1), gamma=0.5)
    decoder_scheduler = torch.optim.lr_scheduler.StepLR(decoder_optimizer, step_size=max(n_epochs//4,1), gamma=0.5)

    # Move models to GPU
    if USE_CUDA:
        stage1_encoder.cuda()
        encoder.cuda()
        decoder.cuda()

    generate_num_ids = []
    for num in generate_nums:
        generate_num_ids.append(output_lang.word2index[num])

    var_num_ids = []
    for var in var_nums:
        if var in output_lang.word2index.keys():
            var_num_ids.append(output_lang.word2index[var])

    best_val_acc = 0
    best_equ_acc = 0
    current_save_dir = os.path.join(save_dir, 'fold-'+str(fold))
    current_best_val_acc = (0,0,0)
    #current_best_train_acc = (0,0,0)
    for epoch in range(n_epochs):
        loss_total = 0
        stage1_loss_total = 0
        random.seed(epoch+random_seed) # for reproduction

        batches_dict = prepare_data_batch(train_pairs, batch_size, lm_tokenizer=bert_tokenizer, use_group_num=False, use_lm=True)

        id_batches = batches_dict['id_batches']
        input_batches = batches_dict['input_batches']
        input_lengths = batches_dict['input_lengths']
        attention_mask_batches = batches_dict['attention_mask_batches']
        token_type_ids_batches = batches_dict['token_type_ids_batches']
        output_batches = batches_dict['output_batches']
        output_lengths = batches_dict['output_lengths']
        nums_batches = batches_dict['nums_batches']
        num_stack_batches = batches_dict['num_stack_batches']
        num_pos_batches = batches_dict['num_pos_batches']
        num_size_batches = batches_dict['num_size_batches']
        ans_batches = batches_dict['ans_batches']
        stage1_span_ids_batches = batches_dict['stage1_span_ids_batches']
        stage1_sentence_ids_batches = batches_dict['stage1_sentence_ids_batches']
        attention_mask_sentence_batches = batches_dict['attention_mask_sentence_batches']
        stage1_span_lengths = batches_dict['stage1_span_lengths']
        sentence_length_batches = batches_dict['sentence_length_batches']

        logs_content = "fold: {}".format(fold+1)
        add_log(log_file, logs_content)
        logs_content = "epoch: {}".format(epoch + 1)
        add_log(log_file, logs_content)
        start = time.time()
        for idx in range(len(input_lengths)):
            step_size = len(input_batches[idx]) // grad_acc_steps
            for step in range(grad_acc_steps):
                start_idx = step * step_size
                end_idx = (step + 1) * step_size
                if step_size == 0:
                    end_idx = len(input_batches[idx])

                if step == grad_acc_steps - 1:
                    grad_acc = False
                else:
                    grad_acc = True

                if step == 0:
                    zero_grad = True
                else:
                    zero_grad = False
                
                
                loss, stage1_loss = train_lm2seq_v2(input_batches[idx][start_idx:end_idx],
                                    input_lengths[idx][start_idx:end_idx],
                                    attention_mask_batches[idx][start_idx:end_idx],
                                    token_type_ids_batches[idx][start_idx:end_idx],
                                    output_batches[idx][start_idx:end_idx],
                                    output_lengths[idx][start_idx:end_idx],
                                    nums_batches[idx][start_idx:end_idx],
                                    num_stack_batches[idx][start_idx:end_idx],
                                    stage1_span_ids_batches[idx][start_idx:end_idx],
                                    stage1_span_lengths[idx][start_idx:end_idx],
                                    stage1_sentence_ids_batches[idx][start_idx:end_idx],
                                    attention_mask_sentence_batches[idx][start_idx:end_idx],
                                    sentence_length_batches[idx][start_idx:end_idx],
                                    id_batches[idx][start_idx:end_idx], 
                                    copy_nums, generate_num_ids, stage1_encoder, encoder, decoder, 
                                    stage1_encoder_optimizer, encoder_optimizer, decoder_optimizer,
                                    output_lang, use_clip=use_clip, clip=gclip, use_teacher_forcing=use_teacher_forcing,
                                    beam_size=1,
                                    grad_acc=grad_acc, zero_grad=zero_grad, grad_acc_steps=grad_acc_steps,
                                    beam_search=beam_search, var_nums=var_num_ids, labels_num=num_labels)
                
                loss_total += loss
                stage1_loss_total += stage1_loss

        stage1_encoder_scheduler.step()
        encoder_scheduler.step()
        decoder_scheduler.step()
        
        logs_content = "stage1 loss: {}".format(stage1_loss_total / len(input_lengths))
        add_log(log_file,logs_content)
        logs_content = "loss: {}".format(loss_total / len(input_lengths))
        add_log(log_file, logs_content)
        logs_content = "training time: {}".format(time_since(time.time() - start))
        add_log(log_file, logs_content)
        logs_content = "--------------------------------"
        add_log(log_file, logs_content)
        
        if epoch > n_epochs - 5:
            train_value_ac = 0
            train_equation_ac = 0
            train_answer_ac = 0
            train_total = 0
            start = time.time()
            for train_batch in train_pairs:
                tokens_dict_train = bert_tokenizer(' '.join(train_batch['input_cell']), add_special_tokens=False)
                raw_input_ids_train = tokens_dict_train["input_ids"]
                tokens_dict_train["input_ids"] = []
                stage1_sentence_ids_train = []
                sentence_length_train = []
                sentence_m_train = [list(group) for k, group in groupby(raw_input_ids_train, lambda x: x == 102 or
                                                                 x == 0) if not k]

                for sentence_id, i in enumerate(sentence_m_train):
                        sentence_length_train.append(len(i)+1)
                        stage1_sentence_ids_train.extend([sentence_id] * (len(i)+1))

                stage1_sentence_ids_train.extend([0] * (raw_input_ids_train.count(0)))

                for t_id in raw_input_ids_train:
                    if t_id == len(bert_tokenizer.vocab):
                        tokens_dict_train["input_ids"].append(1)
                    else:
                        tokens_dict_train["input_ids"].append(t_id)

                num_pos_train = []
                for idx, t_id in enumerate(tokens_dict_train["input_ids"]):
                    if t_id == 1:
                        num_pos_train.append(idx)

                attention_mask_sentence_train = [float(i != -100) for i in train_batch['stage1_span']]
                train_res = evaluate_lm2seq_v2(tokens_dict_train["input_ids"], len(tokens_dict_train["input_ids"]),
                                           tokens_dict_train["attention_mask"], tokens_dict_train["token_type_ids"], 
                                           train_batch['stage1_span'], train_batch['stage1_span_len'],
                                           stage1_sentence_ids_train, attention_mask_sentence_train,
                                           sentence_length_train, train_batch["nums"], copy_nums, generate_num_ids,
                                           stage1_encoder, encoder, decoder,
                                           output_lang, beam_size=beam_size, beam_search=beam_search,
                                           max_length=MAX_OUTPUT_LENGTH, var_nums=var_num_ids, labels_num=num_labels)

                import traceback
                try:
                    train_val_ac, train_equ_ac, train_ans_ac, \
                    train_res_result, train_tar_result = compute_equations_result(train_res,
                                                                                train_batch["output_cell"],
                                                                                output_lang,
                                                                                train_batch["nums"],
                                                                                train_batch['num_stack'],
                                                                                ans_list=train_batch['ans'],
                                                                                tree=False, prefix=False)
                except Exception as e:
                    train_val_ac, train_equ_ac, train_ans_ac = False, False, False

                if train_val_ac:
                    train_value_ac += 1
                if train_equ_ac:
                    train_equation_ac += 1
                if train_ans_ac:
                    train_answer_ac += 1
                train_total += 1

            logs_content = "{}, {}, {}".format(train_equation_ac, train_value_ac, train_total)
            add_log(log_file, logs_content)
            logs_content = "train_answer_acc: {} {}".format(float(train_equation_ac) / train_total, float(train_value_ac) / train_total)
            add_log(log_file, logs_content)
            logs_content = "train eval time: {}".format(time_since(time.time() - start))
            add_log(log_file, logs_content)
            logs_content = "------------------------------------------------------"
            add_log(log_file, logs_content)
            #all_acc_data_train.append((fold, epoch, train_equation_ac, train_value_ac, train_total))
            
        if epoch % 1 == 0 or epoch > n_epochs - 5:
            value_ac = 0
            equation_ac = 0
            answer_ac = 0
            eval_total = 0
            for test_batch in test_pairs:
                # pairs: (id, input_seq, input_len, eq_segs, eq_len, nums, num_pos, num_stack, ans)
                tokens_dict = bert_tokenizer(' '.join(test_batch['input_cell']), add_special_tokens=False)
                raw_input_ids = tokens_dict["input_ids"]
                tokens_dict["input_ids"] = []
                stage1_sentence_ids = []
                sentence_length = []
                sentence_m = [list(group) for k, group in groupby(raw_input_ids, lambda x: x == 102 or
                                                             x == 0) if not k]

                for sentence_id, i in enumerate(sentence_m):
                    sentence_length.append(len(i)+1)
                    stage1_sentence_ids.extend([sentence_id] * (len(i)+1))
   
                stage1_sentence_ids.extend([0] * (raw_input_ids.count(0)))

                for t_id in raw_input_ids:
                    if t_id == len(bert_tokenizer.vocab):
                        tokens_dict["input_ids"].append(1)
                    else:
                        tokens_dict["input_ids"].append(t_id)

                num_pos = []
                for idx, t_id in enumerate(tokens_dict["input_ids"]):
                    if t_id == 1:
                        num_pos.append(idx)
                
                # num_pos = []
                # for idx, t_id in enumerate(tokens_dict["input_ids"]):
                #     if t_id == bert_tokenizer.vocab['[NUM]']:
                #         num_pos.append(idx)
                #stage1_len_max = max(test_batch['stage1_span_len'])
                #stage1_span_ids = pad_seq(test_batch['stage1_span'], test_batch['stage1_span_len'], 
                #                          stage1_len_max, pad_token=-100)


                attention_mask_sentence = [float(i != -100) for i in test_batch['stage1_span']]
                #print(test_batch['id'])
                test_res = evaluate_lm2seq_v2(tokens_dict["input_ids"], len(tokens_dict["input_ids"]),
                                           tokens_dict["attention_mask"], tokens_dict["token_type_ids"], 
                                           test_batch['stage1_span'],
                                           test_batch['stage1_span_len'], stage1_sentence_ids, attention_mask_sentence,
                                           sentence_length, test_batch["nums"], copy_nums, generate_num_ids,
                                           stage1_encoder, encoder, decoder,
                                           output_lang, beam_size=beam_size, beam_search=beam_search,
                                           max_length=MAX_OUTPUT_LENGTH, var_nums=var_num_ids, labels_num=num_labels)

                import traceback
                try:
                    val_ac, equ_ac, ans_ac, \
                    test_res_result, test_tar_result = compute_equations_result(test_res,
                                                                                test_batch["output_cell"],
                                                                                output_lang,
                                                                                test_batch["nums"],
                                                                                test_batch['num_stack'],
                                                                                ans_list=test_batch['ans'],
                                                                                tree=False, prefix=False)
                except Exception as e:
                    # traceback.print_exc()
                    # print(e)
                    val_ac, equ_ac, ans_ac = False, False, False
                if val_ac:
                    value_ac += 1
                if equ_ac:
                    equation_ac += 1
                if ans_ac:
                    answer_ac += 1
                eval_total += 1
                
            logs_content = "{}, {}, {}".format(equation_ac, value_ac, eval_total)
            add_log(log_file, logs_content)
            logs_content = "test_answer_acc: {} {}".format(float(equation_ac) / eval_total, float(value_ac) / eval_total)
            add_log(log_file, logs_content)
            logs_content = "testing time: {}".format(time_since(time.time() - start))
            add_log(log_file, logs_content)
            logs_content = "------------------------------------------------------"
            add_log(log_file, logs_content)
            all_acc_data.append((fold, epoch,equation_ac, value_ac, eval_total))

            torch.save(stage1_encoder.state_dict(), os.path.join(current_save_dir, "lm2seq_stage1_encoder"))
            torch.save(encoder.state_dict(), os.path.join(current_save_dir, "lm2seq_attn_encoder"))
            torch.save(decoder.state_dict(), os.path.join(current_save_dir, "lm2seq_attn_decoder"))

            if best_val_acc < value_ac:
                best_val_acc = value_ac
                current_best_val_acc = (equation_ac, value_ac, eval_total)
                #current_best_train_acc = (train_equation_ac, train_value_ac, train_total)
                torch.save(stage1_encoder.state_dict(), os.path.join(current_save_dir, "lm2seq_stage1_encoder_best_val_acc"))
                torch.save(encoder.state_dict(), os.path.join(current_save_dir, "lm2seq_attn_encoder_best_val_acc"))
                torch.save(decoder.state_dict(), os.path.join(current_save_dir, "lm2seq_attn_decoder_best_val_acc"))
            if best_equ_acc < equation_ac:
                best_equ_acc = equation_ac
                torch.save(stage1_encoder.state_dict(), os.path.join(current_save_dir, "lm2seq_stage1_encoder_best_equ_acc"))
                torch.save(encoder.state_dict(), os.path.join(current_save_dir, "lm2seq_attn_encoder_best_equ_acc"))
                torch.save(decoder.state_dict(), os.path.join(current_save_dir, "lm2seq_attn_decoder_best_equ_acc"))
            if epoch == n_epochs - 1:
                last_acc_fold.append((equation_ac, value_ac, eval_total))
                best_val_acc_fold.append(current_best_val_acc)
                #last_train_acc_fold.append((train_equation_ac, train_value_ac, train_total))
                #best_train_acc_fold.append(current_best_train_acc)

# a, b, c = 0, 0, 0
# for bl in range(len(last_train_acc_fold)):
#     a += last_train_acc_fold[bl][0]
#     b += last_train_acc_fold[bl][1]
#     c += last_train_acc_fold[bl][2]
#     logs_content = "{}".format(last_train_acc_fold[bl])
#     add_log(log_file, logs_content)
# logs_content = "{} {}".format(a / float(c), b / float(c))
# add_log(log_file, logs_content)
# logs_content = "------------------------------------------------------"
# add_log(log_file, logs_content)                
                
a, b, c = 0, 0, 0
for bl in range(len(last_acc_fold)):
    a += last_acc_fold[bl][0]
    b += last_acc_fold[bl][1]
    c += last_acc_fold[bl][2]
    logs_content = "{}".format(last_acc_fold[bl])
    add_log(log_file, logs_content)
logs_content = "{} {}".format(a / float(c), b / float(c))
add_log(log_file, logs_content)
logs_content = "------------------------------------------------------"
add_log(log_file, logs_content)

# a, b, c = 0, 0, 0
# for bl in range(len(best_train_acc_fold)):
#     a += best_train_acc_fold[bl][0]
#     b += best_train_acc_fold[bl][1]
#     c += best_train_acc_fold[bl][2]
#     logs_content = "{}".format(best_train_acc_fold[bl])
#     add_log(log_file, logs_content)
# logs_content = "{} {}".format(a / float(c), b / float(c))
# add_log(log_file, logs_content)
# logs_content = "------------------------------------------------------"
# add_log(log_file, logs_content)

a, b, c = 0, 0, 0
for bl in range(len(best_val_acc_fold)):
    a += best_val_acc_fold[bl][0]
    b += best_val_acc_fold[bl][1]
    c += best_val_acc_fold[bl][2]
    logs_content = "{}".format(best_val_acc_fold[bl])
    add_log(log_file, logs_content)
logs_content = "{} {}".format(a / float(c), b / float(c))
add_log(log_file, logs_content)
logs_content = "------------------------------------------------------"
add_log(log_file, logs_content)

#logs_content = "{}".format(all_acc_data_train)
#add_log(log_file, logs_content)

logs_content = "{}".format(all_acc_data)
add_log(log_file, logs_content)