Example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0212f1bf-c016-40d2-82bb-2b8a064a4a44",
   "metadata": {},
   "source": [
    "# PaD-TS\n",
    "\n",
    "### Load Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "35ed938d-1184-42a3-81f2-36261b12fcef",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/sciclone/home/yli102/.conda/envs/PaD-TS/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import argparse\n",
    "import numpy as np\n",
    "from resample import UniformSampler,Batch_Same_Sampler\n",
    "from Model import PaD_TS\n",
    "from diffmodel_init import create_gaussian_diffusion\n",
    "from training import Trainer\n",
    "from data_preprocessing.real_dataloader import CustomDataset\n",
    "from data_preprocessing.sine_dataloader import SineDataset\n",
    "from data_preprocessing.real_dataloader import fMRIDataset\n",
    "from data_preprocessing.mujoco_dataloader import MuJoCoDataset\n",
    "from torchsummary import summary\n",
    "from data_preprocessing.sampling import sampling\n",
    "from eval_run import discriminative_score,predictive_score,BMMD_score,BMMD_score_naive,VDS_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b689599b-17de-4bb8-a04a-b46e6d0859f5",
   "metadata": {},
   "source": [
    "### Dataset Selection and Args loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a1349593-5978-41f1-a0e7-0ffd450a7c1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = 'sine'\n",
    "if data == 'energy':\n",
    "    from configs.energy_config import Training_args, Model_args, Diffusion_args, DataLoader_args, Data_args\n",
    "elif data == 'stock':\n",
    "    from configs.stock_config import Training_args, Model_args, Diffusion_args, DataLoader_args, Data_args\n",
    "elif data == 'sine':\n",
    "    from configs.sine_config import Training_args, Model_args, Diffusion_args, DataLoader_args, Data_args\n",
    "else:\n",
    "    raise NotImplementedError(f\"Unkown Dataset: {args.data}\")\n",
    "    \n",
    "train_arg = Training_args()\n",
    "model_arg = Model_args()\n",
    "diff_arg = Diffusion_args()\n",
    "dl_arg = DataLoader_args()\n",
    "d_arg = Data_args()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7a9c5222-6644-4a7f-b291-eb4bc63f9fde",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Sampling sine-dataset: 100%|██████████| 10000/10000 [00:00<00:00, 10014.84it/s]\n"
     ]
    }
   ],
   "source": [
    "if data == 'sine':\n",
    "    dataset = SineDataset(\n",
    "            window=24,\n",
    "            num=d_arg.num,\n",
    "            dim=d_arg.dim,\n",
    "            save2npy=d_arg.save2npy,\n",
    "            neg_one_to_one=d_arg.neg_one_to_one,\n",
    "            seed=d_arg.seed,\n",
    "            period=d_arg.period\n",
    "        )\n",
    "else:\n",
    "    dataset = CustomDataset(\n",
    "            name=d_arg.name,\n",
    "            proportion=d_arg.proportion,\n",
    "            data_root=d_arg.data_root,\n",
    "            window=d_arg.window,\n",
    "            save2npy=d_arg.save2npy,\n",
    "            neg_one_to_one=d_arg.neg_one_to_one,\n",
    "            seed=d_arg.seed,\n",
    "            period=d_arg.period)\n",
    "dataloader = torch.utils.data.DataLoader(dataset,\n",
    "                                        batch_size=dl_arg.batch_size,\n",
    "                                        shuffle=dl_arg.shuffle,\n",
    "                                        num_workers=dl_arg.num_workers,\n",
    "                                        drop_last=dl_arg.drop_last,\n",
    "                                        pin_memory=dl_arg.pin_memory)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d5a68ca-bed8-4509-833e-5c2bab291be4",
   "metadata": {},
   "source": [
    "### Model and Diffusion Process init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3336edba-7fa2-4727-bddd-b4128f558b78",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss Function:  MSE_MMD\n",
      "Save Directory:  ./OUTPUT/sine_24/\n",
      "Schedule Sampler:  batch\n",
      "Batch Size:  64\n",
      "Diffusion Steps:  250\n",
      "Epochs:  70000\n",
      "Alpha:  0.0005\n",
      "Window Size:  24\n",
      "Data shape:  (24, 5)\n",
      "Hidden:  128\n"
     ]
    }
   ],
   "source": [
    "model = PaD_TS(hidden_size=model_arg.hidden_size, \n",
    "               num_heads=model_arg.num_heads,\n",
    "               n_encoder=model_arg.n_encoder, \n",
    "               n_decoder=model_arg.n_decoder, \n",
    "               feature_last=model_arg.feature_last, \n",
    "               mlp_ratio=model_arg.mlp_ratio, \n",
    "               input_shape=model_arg.input_shape)\n",
    "diffusion = create_gaussian_diffusion(predict_xstart=diff_arg.predict_xstart,\n",
    "                                      diffusion_steps=diff_arg.diffusion_steps,\n",
    "                                      noise_schedule=diff_arg.noise_schedule,\n",
    "                                      loss=diff_arg.loss,\n",
    "                                      rescale_timesteps=diff_arg.rescale_timesteps)\n",
    "if train_arg.schedule_sampler == 'batch':\n",
    "    schedule_sampler = Batch_Same_Sampler(diffusion)\n",
    "elif train_arg.schedule_sampler == 'uniform':\n",
    "    schedule_sampler = UniformSampler(diffusion)\n",
    "else:\n",
    "    raise NotImplementedError(f\"Unkown sampler: {train_arg.schedule_sampler}\")\n",
    "    \n",
    "trainer = Trainer(model=model,\n",
    "                    diffusion=diffusion, \n",
    "                    data=dataloader, \n",
    "                    batch_size=dl_arg.batch_size,  \n",
    "                    lr = train_arg.lr,\n",
    "                    weight_decay = train_arg.weight_decay,\n",
    "                    lr_anneal_steps=train_arg.lr_anneal_steps,\n",
    "                    log_interval=train_arg.log_interval,\n",
    "                    save_interval=train_arg.save_interval,\n",
    "                    save_dir=train_arg.save_dir,\n",
    "                    schedule_sampler = schedule_sampler,\n",
    "                    mmd_alpha = train_arg.mmd_alpha)\n",
    "print('Loss Function: ',diff_arg.loss)\n",
    "print('Save Directory: ',train_arg.save_dir)\n",
    "print('Schedule Sampler: ',train_arg.schedule_sampler)\n",
    "print('Batch Size: ',dl_arg.batch_size)\n",
    "print('Diffusion Steps: ',diff_arg.diffusion_steps)\n",
    "print('Epochs: ',train_arg.lr_anneal_steps)\n",
    "print('Alpha: ',train_arg.mmd_alpha)\n",
    "print('Window Size: ',d_arg.window)\n",
    "print('Data shape: ',model_arg.input_shape)\n",
    "print('Hidden: ', model_arg.hidden_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cb46a429-0340-40b1-91d7-cb4835b50b0b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======Training======\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "mse: 0.011447, mmd: 0.000006, total: 0.011453: 100%|██████████| 70000/70000 [1:16:48<00:00, 15.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======Done======\n"
     ]
    }
   ],
   "source": [
    "print('======Training======')\n",
    "trainer.train()\n",
    "print('======Done======')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75e8d536-2a41-4445-9694-dd863a1f6140",
   "metadata": {},
   "source": [
    "### Generate samples and Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ed9e68e7-ef96-437b-925f-6fa32885f6e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======Generate Samples======\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 157/157 [02:48<00:00,  1.07s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./OUTPUT/sine_24/ddpm_fake_sine_24.npy\n",
      "======Diff Eval======\n",
      "======Discriminative Score======\n",
      "Fake data: min  0.49786687 , max  1.0\n",
      "Real data: min  0.5000000455140784 , max  0.9999999999995366\n",
      "0\n",
      "WARNING:tensorflow:From /home/yli102/accepted/PaD-TS/eval_utils/discriminative_metric.py:104: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /sciclone/home/yli102/.conda/envs/PaD-TS/lib/python3.8/site-packages/keras/layers/rnn/legacy_cells.py:588: calling Constant.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /sciclone/home/yli102/.conda/envs/PaD-TS/lib/python3.8/site-packages/keras/layers/rnn/legacy_cells.py:602: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /home/yli102/accepted/PaD-TS/eval_utils/discriminative_metric.py:108: all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
      "Instructions for updating:\n",
      "Please use tf.global_variables instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training: 100%|██████████| 2000/2000 [00:36<00:00, 55.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 0:  0.007750000000000035 , 0.4655 , 0.55 \n",
      "\n",
      "1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training: 100%|██████████| 2000/2000 [00:35<00:00, 56.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 1:  0.010000000000000009 , 0.4815 , 0.4985 \n",
      "\n",
      "2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training: 100%|██████████| 2000/2000 [00:34<00:00, 57.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 2:  0.008500000000000008 , 0.531 , 0.452 \n",
      "\n",
      "3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training: 100%|██████████| 2000/2000 [00:34<00:00, 57.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 3:  0.0020000000000000018 , 0.587 , 0.417 \n",
      "\n",
      "4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training: 100%|██████████| 2000/2000 [00:35<00:00, 57.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 4:  0.004750000000000032 , 0.5325 , 0.477 \n",
      "\n",
      "sine:\n",
      "Final Score:  0.006600000000000017 ± 0.003978898672157365\n",
      "======Predictive Score======\n",
      "Fake data: min  0.49786687 , max  1.0\n",
      "Real data: min  0.5000000455140784 , max  0.9999999999995366\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training: 100%|██████████| 5000/5000 [00:51<00:00, 96.85it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0  epoch:  0.09316353265241949 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training: 100%|██████████| 5000/5000 [00:42<00:00, 118.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1  epoch:  0.09282965929533019 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training: 100%|██████████| 5000/5000 [00:44<00:00, 112.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2  epoch:  0.09308739527400377 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training: 100%|██████████| 5000/5000 [00:49<00:00, 101.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3  epoch:  0.09295979806334236 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training: 100%|██████████| 5000/5000 [00:48<00:00, 103.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4  epoch:  0.0927986646338338 \n",
      "\n",
      "sine:\n",
      "Final Score:  0.09296780998378593 ± 0.00019668517967023403\n",
      "======VDS Score======\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5/5 [00:03<00:00,  1.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sine VDS Score: tensor(0.0003)\n",
      "======FDDS Score======\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [00:05<00:00,  2.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sine FDDS Score: tensor(0.0003)\n",
      "======Finished======\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "print('======Generate Samples======')\n",
    "concatenated_tensor = sampling(model,diffusion,dataset.sample_num,dataset.window, dataset.var_num, dl_arg.batch_size)\n",
    "np.save( f'{train_arg.save_dir}ddpm_fake_{d_arg.name}_{dataset.window}.npy', concatenated_tensor.cpu())\n",
    "print(f'{train_arg.save_dir}ddpm_fake_{d_arg.name}_{dataset.window}.npy')\n",
    "\n",
    "print('======Diff Eval======')\n",
    "np_fake = np.array(concatenated_tensor.detach().cpu())\n",
    "print('======Discriminative Score======')\n",
    "discriminative_score(d_arg.name,5, np_fake,length=d_arg.window)\n",
    "print('======Predictive Score======')\n",
    "predictive_score(d_arg.name,5, np_fake,length=d_arg.window)\n",
    "print('======VDS Score======')\n",
    "VDS_score(d_arg.name, concatenated_tensor,length=d_arg.window)\n",
    "print('======FDDS Score======')\n",
    "BMMD_score_naive(d_arg.name, concatenated_tensor,length=d_arg.window)\n",
    "print('======Finished======')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fd69947-f02a-4654-8573-0eeca7d9ac5e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PaD-TS",
   "language": "python",
   "name": "pad-ts"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}