Chapter-07/jigsaw-simple-baseline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8d15c491",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-06-20T21:15:23.452112Z",
     "iopub.status.busy": "2023-06-20T21:15:23.451277Z",
     "iopub.status.idle": "2023-06-20T21:15:31.180987Z",
     "shell.execute_reply": "2023-06-20T21:15:31.179996Z"
    },
    "papermill": {
     "duration": 7.738306,
     "end_time": "2023-06-20T21:15:31.183378",
     "exception": false,
     "start_time": "2023-06-20T21:15:23.445072",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/__init__.py:98: UserWarning: unable to load libtensorflow_io_plugins.so: unable to open file: libtensorflow_io_plugins.so, from paths: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io_plugins.so']\n",
      "caused by: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io_plugins.so: undefined symbol: _ZN3tsl6StatusC1EN10tensorflow5error4CodeESt17basic_string_viewIcSt11char_traitsIcEENS_14SourceLocationE']\n",
      "  warnings.warn(f\"unable to load libtensorflow_io_plugins.so: {e}\")\n",
      "/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/__init__.py:104: UserWarning: file system plugins are not loaded: unable to open file: libtensorflow_io.so, from paths: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io.so']\n",
      "caused by: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io.so: undefined symbol: _ZTVN10tensorflow13GcsFileSystemE']\n",
      "  warnings.warn(f\"file system plugins are not loaded: {e}\")\n"
     ]
    }
   ],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "import os\n",
    "import gc\n",
    "import logging\n",
    "import datetime\n",
    "import warnings\n",
    "import pickle\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler\n",
    "from keras.layers import Input, Dense, Embedding, SpatialDropout1D, Dropout, add, concatenate\n",
    "from keras.layers import CuDNNLSTM, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D\n",
    "from keras.preprocessing import text, sequence\n",
    "from keras.losses import binary_crossentropy\n",
    "from keras import backend as K\n",
    "import keras.layers as L\n",
    "from keras import initializers, regularizers, constraints, optimizers, layers\n",
    "\n",
    "from keras.models import Model\n",
    "from keras.optimizers import Adam\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.model_selection import KFold\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils import pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f732de23",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-20T21:15:31.194888Z",
     "iopub.status.busy": "2023-06-20T21:15:31.194283Z",
     "iopub.status.idle": "2023-06-20T21:15:31.200068Z",
     "shell.execute_reply": "2023-06-20T21:15:31.199085Z"
    },
    "papermill": {
     "duration": 0.013965,
     "end_time": "2023-06-20T21:15:31.202406",
     "exception": false,
     "start_time": "2023-06-20T21:15:31.188441",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "DEBUG = False\n",
    "COMMENT_TEXT_COL = 'comment_text'\n",
    "EMB_MAX_FEAT = 300\n",
    "MAX_LEN = 220\n",
    "MAX_FEATURES = 100000\n",
    "BATCH_SIZE = 512\n",
    "NUM_EPOCHS = 4\n",
    "LSTM_UNITS = 128\n",
    "DENSE_HIDDEN_UNITS = 512\n",
    "NUM_MODELS = 1\n",
    "EMB_PATHS = [\n",
    "    '../input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',\n",
    "    '../input/glove840b300dtxt/glove.840B.300d.txt'\n",
    "]\n",
    "JIGSAW_PATH = '../input/jigsaw-unintended-bias-in-toxicity-classification/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7d89522a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-20T21:15:31.212640Z",
     "iopub.status.busy": "2023-06-20T21:15:31.212364Z",
     "iopub.status.idle": "2023-06-20T21:15:31.218256Z",
     "shell.execute_reply": "2023-06-20T21:15:31.217296Z"
    },
    "papermill": {
     "duration": 0.013394,
     "end_time": "2023-06-20T21:15:31.220380",
     "exception": false,
     "start_time": "2023-06-20T21:15:31.206986",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_logger():\n",
    "    '''\n",
    "        Logger\n",
    "    '''\n",
    "    FORMAT = '[%(levelname)s]%(asctime)s:%(name)s:%(message)s'\n",
    "    logging.basicConfig(format=FORMAT)\n",
    "    logger = logging.getLogger('main')\n",
    "    logger.setLevel(logging.DEBUG)\n",
    "    return logger\n",
    "    \n",
    "logger = get_logger()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fe507f3f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-20T21:15:31.230138Z",
     "iopub.status.busy": "2023-06-20T21:15:31.229891Z",
     "iopub.status.idle": "2023-06-20T21:15:31.237407Z",
     "shell.execute_reply": "2023-06-20T21:15:31.236550Z"
    },
    "papermill": {
     "duration": 0.014808,
     "end_time": "2023-06-20T21:15:31.239512",
     "exception": false,
     "start_time": "2023-06-20T21:15:31.224704",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_coefs(word, *arr):\n",
    "    return word, np.asarray(arr, dtype='float32')\n",
    "\n",
    "def load_embeddings(path):\n",
    "    with open(path) as f:\n",
    "        return dict(get_coefs(*line.strip().split(' ')) for line in f)\n",
    "\n",
    "def build_embedding_matrix(word_index, path):\n",
    "    '''\n",
    "     Build embeddings\n",
    "    '''\n",
    "    logger.info('Build embedding matrix')\n",
    "    embedding_index = load_embeddings(path)\n",
    "    embedding_matrix = np.zeros((len(word_index) + 1, EMB_MAX_FEAT))\n",
    "    for word, i in word_index.items():\n",
    "        try:\n",
    "            embedding_matrix[i] = embedding_index[word]\n",
    "        except KeyError:\n",
    "            pass\n",
    "        except:\n",
    "            embedding_matrix[i] = embeddings_index[\"unknown\"]\n",
    "            \n",
    "    del embedding_index\n",
    "    gc.collect()\n",
    "    return embedding_matrix\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0998e044",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-20T21:15:31.249846Z",
     "iopub.status.busy": "2023-06-20T21:15:31.249558Z",
     "iopub.status.idle": "2023-06-20T21:15:31.254022Z",
     "shell.execute_reply": "2023-06-20T21:15:31.253125Z"
    },
    "papermill": {
     "duration": 0.011654,
     "end_time": "2023-06-20T21:15:31.255901",
     "exception": false,
     "start_time": "2023-06-20T21:15:31.244247",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def custom_loss(y_true, y_pred):\n",
    "    return binary_crossentropy(K.reshape(y_true[:,0],(-1,1)), y_pred) * y_true[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c6c7d144",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-20T21:15:31.265687Z",
     "iopub.status.busy": "2023-06-20T21:15:31.265413Z",
     "iopub.status.idle": "2023-06-20T21:15:31.270909Z",
     "shell.execute_reply": "2023-06-20T21:15:31.269937Z"
    },
    "papermill": {
     "duration": 0.012882,
     "end_time": "2023-06-20T21:15:31.273171",
     "exception": false,
     "start_time": "2023-06-20T21:15:31.260289",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def load_data():\n",
    "    logger.info('Load train and test data')\n",
    "    train = pd.read_csv(os.path.join(JIGSAW_PATH,'train.csv'), index_col='id')\n",
    "    if DEBUG:\n",
    "        train = train.sample(100_000)\n",
    "    test = pd.read_csv(os.path.join(JIGSAW_PATH,'test.csv'), index_col='id')\n",
    "    print(train.shape, test.shape)\n",
    "    return train, test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3c8368e1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-20T21:15:31.283170Z",
     "iopub.status.busy": "2023-06-20T21:15:31.282897Z",
     "iopub.status.idle": "2023-06-20T21:15:31.291149Z",
     "shell.execute_reply": "2023-06-20T21:15:31.290344Z"
    },
    "papermill": {
     "duration": 0.015451,
     "end_time": "2023-06-20T21:15:31.293158",
     "exception": false,
     "start_time": "2023-06-20T21:15:31.277707",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "def perform_preprocessing(train, test):\n",
    "    '''\n",
    "        credits to: https://www.kaggle.com/artgor/cnn-in-keras-on-folds\n",
    "        credits to: https://www.kaggle.com/taindow/simple-cudnngru-python-keras\n",
    "    '''\n",
    "    logger.info('data preprocessing')\n",
    "    punct_mapping = {\"_\":\" \", \"`\":\" \"}\n",
    "    punct = \"/-'?!.,#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—–&'\n",
    "    punct += '©^®` <→°€™› ♥←×§″′Â█½à…“★”–●â►−¢²¬░¶↑±¿▾═¦║―¥▓—‹─▒：¼⊕▼▪†■’▀¨▄♫☆é¯♦¤▲è¸¾Ã⋅‘∞∙）↓、│（»，♪╩╚³・╦╣╔╗▬❤ïØ¹≤‡√'\n",
    "    def clean_special_chars(text, punct, mapping):\n",
    "        for p in mapping:\n",
    "            text = text.replace(p, mapping[p])    \n",
    "        for p in punct:\n",
    "            text = text.replace(p, f' {p} ')     \n",
    "        return text\n",
    "\n",
    "    for df in [train, test]:\n",
    "        df[COMMENT_TEXT_COL] = df[COMMENT_TEXT_COL].astype(str)\n",
    "        df[COMMENT_TEXT_COL] = df[COMMENT_TEXT_COL].apply(lambda x: clean_special_chars(x, punct, punct_mapping))\n",
    "    \n",
    "    return train, test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0ffa7b1c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-20T21:15:31.303006Z",
     "iopub.status.busy": "2023-06-20T21:15:31.302748Z",
     "iopub.status.idle": "2023-06-20T21:15:31.316535Z",
     "shell.execute_reply": "2023-06-20T21:15:31.315614Z"
    },
    "papermill": {
     "duration": 0.021211,
     "end_time": "2023-06-20T21:15:31.318758",
     "exception": false,
     "start_time": "2023-06-20T21:15:31.297547",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def run_proc_and_tokenizer(train, test):\n",
    "    '''\n",
    "        Run proc and tokenizer\n",
    "    '''\n",
    "    logger.info('Running processing and tokenizer')\n",
    " \n",
    "    identity_columns = ['asian', 'atheist',\n",
    "       'bisexual', 'black', 'buddhist', 'christian', 'female',\n",
    "       'heterosexual', 'hindu', 'homosexual_gay_or_lesbian',\n",
    "       'intellectual_or_learning_disability', 'jewish', 'latino', 'male',\n",
    "       'muslim', 'other_disability', 'other_gender',\n",
    "       'other_race_or_ethnicity', 'other_religion',\n",
    "       'other_sexual_orientation', 'physical_disability',\n",
    "       'psychiatric_or_mental_illness', 'transgender', 'white']\n",
    "       \n",
    "    # Overall\n",
    "    weights = np.ones((len(train),)) / 4\n",
    "    # Subgroup\n",
    "    weights += (train[identity_columns].fillna(0).values>=0.5).sum(axis=1).astype(bool).astype(np.int) / 4\n",
    "    # Background Positive, Subgroup Negative\n",
    "    weights += (( (train['target'].values>=0.5).astype(bool).astype(np.int) +\n",
    "       (train[identity_columns].fillna(0).values<0.5).sum(axis=1).astype(bool).astype(np.int) ) > 1 ).astype(bool).astype(np.int) / 4\n",
    "    # Background Negative, Subgroup Positive\n",
    "    weights += (( (train['target'].values<0.5).astype(bool).astype(np.int) +\n",
    "       (train[identity_columns].fillna(0).values>=0.5).sum(axis=1).astype(bool).astype(np.int) ) > 1 ).astype(bool).astype(np.int) / 4\n",
    "    loss_weight = 1.0 / weights.mean()\n",
    "    \n",
    "    y_train = np.vstack([(train['target'].values>=0.5).astype(np.int),weights]).T\n",
    "    y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']].values\n",
    "    \n",
    "    logger.info('Fitting tokenizer')\n",
    "    tokenizer = Tokenizer() \n",
    "    tokenizer.fit_on_texts(list(train[COMMENT_TEXT_COL]) + list(test[COMMENT_TEXT_COL]))\n",
    "    word_index = tokenizer.word_index\n",
    "    X_train = tokenizer.texts_to_sequences(list(train[COMMENT_TEXT_COL]))\n",
    "    X_test = tokenizer.texts_to_sequences(list(test[COMMENT_TEXT_COL]))\n",
    "    X_train = pad_sequences(X_train, maxlen=MAX_LEN)\n",
    "    X_test = pad_sequences(X_test, maxlen=MAX_LEN)\n",
    "    \n",
    "    with open('temporary.pickle', mode='wb') as f:\n",
    "        pickle.dump(X_test, f) # use temporary file to reduce memory\n",
    "\n",
    "    del identity_columns, weights, tokenizer, train, test\n",
    "    gc.collect()\n",
    "    \n",
    "    return X_train, y_train, y_aux_train, word_index, loss_weight\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a6d69ad5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-20T21:15:31.329554Z",
     "iopub.status.busy": "2023-06-20T21:15:31.328846Z",
     "iopub.status.idle": "2023-06-20T21:15:31.333817Z",
     "shell.execute_reply": "2023-06-20T21:15:31.332933Z"
    },
    "papermill": {
     "duration": 0.012706,
     "end_time": "2023-06-20T21:15:31.335877",
     "exception": false,
     "start_time": "2023-06-20T21:15:31.323171",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def build_embeddings(word_index):\n",
    "    '''\n",
    "     Build embeddings\n",
    "    '''\n",
    "    logger.info('Load and build embeddings')\n",
    "    embedding_matrix = np.concatenate(\n",
    "        [build_embedding_matrix(word_index, f) for f in EMB_PATHS], axis=-1) \n",
    "    return embedding_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b2a4b950",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-20T21:15:31.345701Z",
     "iopub.status.busy": "2023-06-20T21:15:31.345425Z",
     "iopub.status.idle": "2023-06-20T21:15:31.353481Z",
     "shell.execute_reply": "2023-06-20T21:15:31.352516Z"
    },
    "papermill": {
     "duration": 0.01553,
     "end_time": "2023-06-20T21:15:31.355749",
     "exception": false,
     "start_time": "2023-06-20T21:15:31.340219",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def build_model(embedding_matrix, num_aux_targets, loss_weight):\n",
    "    '''\n",
    "        Build model\n",
    "    '''\n",
    "    logger.info('Build model')\n",
    "    words = Input(shape=(MAX_LEN,))\n",
    "    x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False)(words)\n",
    "    x = SpatialDropout1D(0.3)(x)\n",
    "    x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)\n",
    "    x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)\n",
    "\n",
    "    hidden = concatenate([GlobalMaxPooling1D()(x),GlobalAveragePooling1D()(x),])\n",
    "    hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu')(hidden)])\n",
    "    hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu')(hidden)])\n",
    "    result = Dense(1, activation='sigmoid')(hidden)\n",
    "    aux_result = Dense(num_aux_targets, activation='sigmoid')(hidden)\n",
    "    \n",
    "    model = Model(inputs=words, outputs=[result, aux_result])\n",
    "    model.compile(loss=[custom_loss,'binary_crossentropy'], loss_weights=[loss_weight, 1.0], optimizer='adam')\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "491b533c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-20T21:15:31.365581Z",
     "iopub.status.busy": "2023-06-20T21:15:31.365332Z",
     "iopub.status.idle": "2023-06-20T21:15:31.373115Z",
     "shell.execute_reply": "2023-06-20T21:15:31.372293Z"
    },
    "papermill": {
     "duration": 0.014851,
     "end_time": "2023-06-20T21:15:31.375033",
     "exception": false,
     "start_time": "2023-06-20T21:15:31.360182",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def run_model(X_train, y_train, y_aux_train, embedding_matrix, word_index, loss_weight):\n",
    "    '''\n",
    "        Run model\n",
    "    '''\n",
    "    logger.info('Run model')\n",
    "    \n",
    "    checkpoint_predictions = []\n",
    "    weights = []\n",
    "    for model_idx in range(NUM_MODELS):\n",
    "        model = build_model(embedding_matrix, y_aux_train.shape[-1], loss_weight)\n",
    "        for global_epoch in range(NUM_EPOCHS):\n",
    "            model.fit(\n",
    "                X_train, [y_train, y_aux_train],\n",
    "                batch_size=BATCH_SIZE, epochs=1, verbose=1,\n",
    "                callbacks=[LearningRateScheduler(lambda epoch: 1.1e-3 * (0.55 ** global_epoch))]\n",
    "            )\n",
    "            with open('temporary.pickle', mode='rb') as f:\n",
    "                X_test = pickle.load(f) # use temporary file to reduce memory\n",
    "            checkpoint_predictions.append(model.predict(X_test, batch_size=1024)[0].flatten())\n",
    "            del X_test\n",
    "            gc.collect()\n",
    "            weights.append(2 ** global_epoch)\n",
    "        del model\n",
    "        gc.collect()\n",
    "    \n",
    "    preds = np.average(checkpoint_predictions, weights=weights, axis=0)\n",
    "    return preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "426f2e0b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-20T21:15:31.385203Z",
     "iopub.status.busy": "2023-06-20T21:15:31.384470Z",
     "iopub.status.idle": "2023-06-20T21:15:31.390040Z",
     "shell.execute_reply": "2023-06-20T21:15:31.389199Z"
    },
    "papermill": {
     "duration": 0.012633,
     "end_time": "2023-06-20T21:15:31.392039",
     "exception": false,
     "start_time": "2023-06-20T21:15:31.379406",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def submit(sub_preds):\n",
    "    logger.info('Prepare submission')\n",
    "    submission = pd.read_csv(os.path.join(JIGSAW_PATH,'sample_submission.csv'), index_col='id')\n",
    "    submission['prediction'] = sub_preds\n",
    "    submission.reset_index(drop=False, inplace=True)\n",
    "    submission.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ebd6380f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-20T21:15:31.402592Z",
     "iopub.status.busy": "2023-06-20T21:15:31.401801Z",
     "iopub.status.idle": "2023-06-20T21:15:31.407382Z",
     "shell.execute_reply": "2023-06-20T21:15:31.406512Z"
    },
    "papermill": {
     "duration": 0.012785,
     "end_time": "2023-06-20T21:15:31.409269",
     "exception": false,
     "start_time": "2023-06-20T21:15:31.396484",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def main():\n",
    "    train, test = load_data()\n",
    "    train, test = perform_preprocessing(train, test)\n",
    "    X_train, y_train, y_aux_train, word_index, loss_weight = run_proc_and_tokenizer(train, test)\n",
    "    embedding_matrix = build_embeddings(word_index)\n",
    "    sub_preds = run_model(X_train, y_train, y_aux_train, embedding_matrix, word_index, loss_weight)\n",
    "    submit(sub_preds)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e90a01c2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-20T21:15:31.419433Z",
     "iopub.status.busy": "2023-06-20T21:15:31.418715Z",
     "iopub.status.idle": "2023-06-20T22:29:41.215701Z",
     "shell.execute_reply": "2023-06-20T22:29:41.214677Z"
    },
    "papermill": {
     "duration": 4449.804974,
     "end_time": "2023-06-20T22:29:41.218544",
     "exception": false,
     "start_time": "2023-06-20T21:15:31.413570",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1804874, 44) (97320, 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_22/3081288797.py:19: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  weights += (train[identity_columns].fillna(0).values>=0.5).sum(axis=1).astype(bool).astype(np.int) / 4\n",
      "/tmp/ipykernel_22/3081288797.py:21: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  weights += (( (train['target'].values>=0.5).astype(bool).astype(np.int) +\n",
      "/tmp/ipykernel_22/3081288797.py:22: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  (train[identity_columns].fillna(0).values<0.5).sum(axis=1).astype(bool).astype(np.int) ) > 1 ).astype(bool).astype(np.int) / 4\n",
      "/tmp/ipykernel_22/3081288797.py:24: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  weights += (( (train['target'].values<0.5).astype(bool).astype(np.int) +\n",
      "/tmp/ipykernel_22/3081288797.py:25: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  (train[identity_columns].fillna(0).values>=0.5).sum(axis=1).astype(bool).astype(np.int) ) > 1 ).astype(bool).astype(np.int) / 4\n",
      "/tmp/ipykernel_22/3081288797.py:28: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  y_train = np.vstack([(train['target'].values>=0.5).astype(np.int),weights]).T\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3526/3526 [==============================] - 819s 229ms/step - loss: 0.2979 - dense_2_loss: 0.0596 - dense_3_loss: 0.1076 - lr: 0.0011\n",
      "96/96 [==============================] - 18s 164ms/step\n",
      "3526/3526 [==============================] - 810s 230ms/step - loss: 0.2701 - dense_2_loss: 0.0524 - dense_3_loss: 0.1028 - lr: 6.0500e-04\n",
      "96/96 [==============================] - 16s 163ms/step\n",
      "3526/3526 [==============================] - 811s 230ms/step - loss: 0.2610 - dense_2_loss: 0.0498 - dense_3_loss: 0.1019 - lr: 3.3275e-04\n",
      "96/96 [==============================] - 16s 162ms/step\n",
      "3526/3526 [==============================] - 811s 230ms/step - loss: 0.2546 - dense_2_loss: 0.0480 - dense_3_loss: 0.1014 - lr: 1.8301e-04\n",
      "96/96 [==============================] - 16s 163ms/step\n"
     ]
    }
   ],
   "source": [
    "main()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 4474.010804,
   "end_time": "2023-06-20T22:29:46.154224",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-06-20T21:15:12.143420",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}