DNN_tf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello,world\n",
      "{'FolderName': 'fitnd/163391/', 'input_dim': 5, 'epsion': 0.1, 'output_dim': 1, 'ActFuc': 1, 'hidden_units': [200], 'astddev': 0.22360679774997896, 'bstddev': 0.22360679774997896, 'ASI': 0, 'learning_rate': 1e-06, 'learning_rateDecay': 2e-07, 'seed': 0, 'train_size': 1000, 'batch_size': 1000, 'test_size': 1000, 'x_start': -1.5707963267948966, 'x_end': 1.5707963267948966, 'tol': -10000000000.0, 'Total_Step': 600000}\n",
      "input x_data is:  [[-1.2009458   1.07467463 -1.45438227  1.08224157  0.69630963]\n",
      " [ 0.40130579 -0.74422722  1.46257397  1.09490704  0.19977801]\n",
      " [-0.59023423  0.63400853  0.74087544  1.47139976  0.62839082]\n",
      " ...\n",
      " [-1.21872939 -0.99295564  1.29511923  0.10423963  0.95627097]\n",
      " [ 0.58990383 -0.08376098  0.3837747  -1.47459266  1.0836172 ]\n",
      " [-0.56531571 -0.8721598   1.17983505  0.73618282  1.3689457 ]]\n",
      "input x_data is:  [[-0.19614916  1.32540011  1.51555963  1.55043265  0.77850862]\n",
      " [ 0.91861452  0.96602054 -1.00772039 -0.47731773 -1.08874494]\n",
      " [-0.48440708  0.97342069  1.38595082  0.64845117 -0.87421128]\n",
      " ...\n",
      " [ 1.46119728 -0.03204339  0.09717062 -1.41219764  1.45361895]\n",
      " [ 0.62875617 -1.28409289 -0.21840802 -0.65277154  0.74816317]\n",
      " [ 1.42266597  0.30594333 -1.13014199  0.59427012 -0.43916541]]\n",
      "add_hidden: [5, 200]\n",
      "input_dim:5, output_dim:200\n",
      "tanh\n",
      "INFO:tensorflow:Restoring parameters from fitnd/163391/model/model.ckpt\n",
      "initial 3.7699757\n",
      "time elapse: 0.278\n",
      "model, epoch: 0, test loss: 3.055187\n",
      "model, epoch: 0, train loss: 3.181990\n",
      "time elapse: 1.869\n",
      "model, epoch: 500, test loss: 2.927019\n",
      "model, epoch: 500, train loss: 3.049088\n",
      "time elapse: 3.234\n",
      "model, epoch: 1000, test loss: 2.804497\n",
      "model, epoch: 1000, train loss: 2.921877\n",
      "time elapse: 4.738\n",
      "model, epoch: 1500, test loss: 2.687076\n",
      "model, epoch: 1500, train loss: 2.799848\n",
      "time elapse: 6.201\n",
      "model, epoch: 2000, test loss: 2.574486\n",
      "model, epoch: 2000, train loss: 2.682730\n",
      "time elapse: 7.692\n",
      "model, epoch: 2500, test loss: 2.466411\n",
      "model, epoch: 2500, train loss: 2.570191\n",
      "time elapse: 9.204\n",
      "model, epoch: 3000, test loss: 2.362536\n",
      "model, epoch: 3000, train loss: 2.461898\n",
      "time elapse: 10.691\n",
      "model, epoch: 3500, test loss: 2.262718\n",
      "model, epoch: 3500, train loss: 2.357714\n",
      "time elapse: 12.273\n",
      "model, epoch: 4000, test loss: 2.166798\n",
      "model, epoch: 4000, train loss: 2.257484\n",
      "time elapse: 13.811\n",
      "model, epoch: 4500, test loss: 2.074642\n",
      "model, epoch: 4500, train loss: 2.161086\n",
      "time elapse: 15.389\n",
      "model, epoch: 5000, test loss: 1.986143\n",
      "model, epoch: 5000, train loss: 2.068418\n",
      "time elapse: 17.006\n",
      "model, epoch: 5500, test loss: 1.901224\n",
      "model, epoch: 5500, train loss: 1.979402\n",
      "time elapse: 18.493\n",
      "model, epoch: 6000, test loss: 1.819790\n",
      "model, epoch: 6000, train loss: 1.893939\n",
      "time elapse: 20.012\n",
      "model, epoch: 6500, test loss: 1.741757\n",
      "model, epoch: 6500, train loss: 1.811946\n",
      "time elapse: 21.512\n",
      "model, epoch: 7000, test loss: 1.667080\n",
      "model, epoch: 7000, train loss: 1.733381\n",
      "time elapse: 23.009\n",
      "model, epoch: 7500, test loss: 1.595730\n",
      "model, epoch: 7500, train loss: 1.658218\n",
      "time elapse: 24.605\n",
      "model, epoch: 8000, test loss: 1.527652\n",
      "model, epoch: 8000, train loss: 1.586399\n",
      "time elapse: 26.151\n",
      "model, epoch: 8500, test loss: 1.462755\n",
      "model, epoch: 8500, train loss: 1.517837\n",
      "time elapse: 27.737\n",
      "model, epoch: 9000, test loss: 1.400975\n",
      "model, epoch: 9000, train loss: 1.452475\n",
      "time elapse: 29.295\n",
      "model, epoch: 9500, test loss: 1.342240\n",
      "model, epoch: 9500, train loss: 1.390249\n",
      "time elapse: 30.911\n",
      "model, epoch: 10000, test loss: 1.286478\n",
      "model, epoch: 10000, train loss: 1.331089\n",
      "time elapse: 32.637\n",
      "model, epoch: 10500, test loss: 1.233597\n",
      "model, epoch: 10500, train loss: 1.274909\n",
      "time elapse: 35.126\n",
      "model, epoch: 11000, test loss: 1.183543\n",
      "model, epoch: 11000, train loss: 1.221658\n",
      "time elapse: 36.894\n",
      "model, epoch: 11500, test loss: 1.136275\n",
      "model, epoch: 11500, train loss: 1.171285\n",
      "time elapse: 38.427\n",
      "model, epoch: 12000, test loss: 1.091720\n",
      "model, epoch: 12000, train loss: 1.123721\n",
      "time elapse: 40.037\n",
      "model, epoch: 12500, test loss: 1.049806\n",
      "model, epoch: 12500, train loss: 1.078905\n",
      "time elapse: 41.705\n",
      "model, epoch: 13000, test loss: 1.010454\n",
      "model, epoch: 13000, train loss: 1.036760\n",
      "time elapse: 43.541\n",
      "model, epoch: 13500, test loss: 0.973570\n",
      "model, epoch: 13500, train loss: 0.997201\n",
      "time elapse: 45.211\n",
      "model, epoch: 14000, test loss: 0.939048\n",
      "model, epoch: 14000, train loss: 0.960139\n",
      "time elapse: 47.002\n",
      "model, epoch: 14500, test loss: 0.906809\n",
      "model, epoch: 14500, train loss: 0.925506\n",
      "time elapse: 48.578\n",
      "model, epoch: 15000, test loss: 0.876749\n",
      "model, epoch: 15000, train loss: 0.893209\n",
      "time elapse: 50.190\n",
      "model, epoch: 15500, test loss: 0.848768\n",
      "model, epoch: 15500, train loss: 0.863149\n",
      "time elapse: 51.799\n",
      "model, epoch: 16000, test loss: 0.822806\n",
      "model, epoch: 16000, train loss: 0.835261\n",
      "time elapse: 53.594\n",
      "model, epoch: 16500, test loss: 0.798799\n",
      "model, epoch: 16500, train loss: 0.809466\n",
      "time elapse: 55.379\n",
      "model, epoch: 17000, test loss: 0.776717\n",
      "model, epoch: 17000, train loss: 0.785725\n",
      "time elapse: 57.164\n",
      "model, epoch: 17500, test loss: 0.756503\n",
      "model, epoch: 17500, train loss: 0.763970\n",
      "time elapse: 58.940\n",
      "model, epoch: 18000, test loss: 0.738120\n",
      "model, epoch: 18000, train loss: 0.744150\n",
      "time elapse: 60.819\n",
      "model, epoch: 18500, test loss: 0.721509\n",
      "model, epoch: 18500, train loss: 0.726194\n",
      "time elapse: 62.732\n",
      "model, epoch: 19000, test loss: 0.706649\n",
      "model, epoch: 19000, train loss: 0.710068\n",
      "time elapse: 64.415\n",
      "model, epoch: 19500, test loss: 0.693485\n",
      "model, epoch: 19500, train loss: 0.695701\n",
      "time elapse: 66.098\n",
      "model, epoch: 20000, test loss: 0.681944\n",
      "model, epoch: 20000, train loss: 0.682998\n",
      "time elapse: 67.784\n",
      "model, epoch: 20500, test loss: 0.671962\n",
      "model, epoch: 20500, train loss: 0.671887\n",
      "time elapse: 69.516\n",
      "model, epoch: 21000, test loss: 0.663442\n",
      "model, epoch: 21000, train loss: 0.662258\n",
      "time elapse: 71.282\n",
      "model, epoch: 21500, test loss: 0.656290\n",
      "model, epoch: 21500, train loss: 0.654016\n",
      "time elapse: 73.050\n",
      "model, epoch: 22000, test loss: 0.650402\n",
      "model, epoch: 22000, train loss: 0.647050\n",
      "time elapse: 74.826\n",
      "model, epoch: 22500, test loss: 0.645665\n",
      "model, epoch: 22500, train loss: 0.641250\n",
      "time elapse: 76.559\n",
      "model, epoch: 23000, test loss: 0.641961\n",
      "model, epoch: 23000, train loss: 0.636501\n",
      "time elapse: 78.365\n",
      "model, epoch: 23500, test loss: 0.639178\n",
      "model, epoch: 23500, train loss: 0.632691\n",
      "time elapse: 80.247\n",
      "model, epoch: 24000, test loss: 0.637193\n",
      "model, epoch: 24000, train loss: 0.629700\n",
      "time elapse: 82.014\n",
      "model, epoch: 24500, test loss: 0.635874\n",
      "model, epoch: 24500, train loss: 0.627411\n",
      "time elapse: 83.765\n",
      "model, epoch: 25000, test loss: 0.635081\n",
      "model, epoch: 25000, train loss: 0.625701\n",
      "time elapse: 85.668\n",
      "model, epoch: 25500, test loss: 0.634665\n",
      "model, epoch: 25500, train loss: 0.624436\n",
      "time elapse: 87.615\n",
      "model, epoch: 26000, test loss: 0.634481\n",
      "model, epoch: 26000, train loss: 0.623485\n",
      "time elapse: 89.421\n",
      "model, epoch: 26500, test loss: 0.634401\n",
      "model, epoch: 26500, train loss: 0.622726\n",
      "time elapse: 91.342\n",
      "model, epoch: 27000, test loss: 0.634331\n",
      "model, epoch: 27000, train loss: 0.622061\n",
      "time elapse: 93.202\n",
      "model, epoch: 27500, test loss: 0.634222\n",
      "model, epoch: 27500, train loss: 0.621426\n",
      "time elapse: 95.187\n",
      "model, epoch: 28000, test loss: 0.634069\n",
      "model, epoch: 28000, train loss: 0.620790\n",
      "time elapse: 97.035\n",
      "model, epoch: 28500, test loss: 0.633889\n",
      "model, epoch: 28500, train loss: 0.620144\n",
      "time elapse: 98.995\n",
      "model, epoch: 29000, test loss: 0.633696\n",
      "model, epoch: 29000, train loss: 0.619486\n",
      "time elapse: 100.890\n",
      "model, epoch: 29500, test loss: 0.633501\n",
      "model, epoch: 29500, train loss: 0.618815\n",
      "time elapse: 102.785\n",
      "model, epoch: 30000, test loss: 0.633308\n",
      "model, epoch: 30000, train loss: 0.618136\n",
      "time elapse: 104.804\n",
      "model, epoch: 30500, test loss: 0.633120\n",
      "model, epoch: 30500, train loss: 0.617449\n",
      "time elapse: 106.916\n",
      "model, epoch: 31000, test loss: 0.632938\n",
      "model, epoch: 31000, train loss: 0.616757\n",
      "time elapse: 108.794\n",
      "model, epoch: 31500, test loss: 0.632764\n",
      "model, epoch: 31500, train loss: 0.616062\n",
      "time elapse: 111.049\n",
      "model, epoch: 32000, test loss: 0.632598\n",
      "model, epoch: 32000, train loss: 0.615366\n",
      "time elapse: 113.286\n",
      "model, epoch: 32500, test loss: 0.632441\n",
      "model, epoch: 32500, train loss: 0.614672\n",
      "time elapse: 115.695\n",
      "model, epoch: 33000, test loss: 0.632291\n",
      "model, epoch: 33000, train loss: 0.613982\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time elapse: 117.740\n",
      "model, epoch: 33500, test loss: 0.632148\n",
      "model, epoch: 33500, train loss: 0.613297\n",
      "time elapse: 119.856\n",
      "model, epoch: 34000, test loss: 0.632012\n",
      "model, epoch: 34000, train loss: 0.612618\n",
      "time elapse: 121.905\n",
      "model, epoch: 34500, test loss: 0.631882\n",
      "model, epoch: 34500, train loss: 0.611946\n",
      "time elapse: 123.900\n",
      "model, epoch: 35000, test loss: 0.631760\n",
      "model, epoch: 35000, train loss: 0.611283\n",
      "time elapse: 126.163\n",
      "model, epoch: 35500, test loss: 0.631646\n",
      "model, epoch: 35500, train loss: 0.610628\n",
      "time elapse: 128.412\n",
      "model, epoch: 36000, test loss: 0.631537\n",
      "model, epoch: 36000, train loss: 0.609981\n",
      "time elapse: 130.651\n",
      "model, epoch: 36500, test loss: 0.631435\n",
      "model, epoch: 36500, train loss: 0.609344\n",
      "time elapse: 132.977\n",
      "model, epoch: 37000, test loss: 0.631340\n",
      "model, epoch: 37000, train loss: 0.608715\n",
      "time elapse: 135.271\n",
      "model, epoch: 37500, test loss: 0.631251\n",
      "model, epoch: 37500, train loss: 0.608095\n",
      "time elapse: 137.517\n",
      "model, epoch: 38000, test loss: 0.631171\n",
      "model, epoch: 38000, train loss: 0.607484\n",
      "time elapse: 139.790\n",
      "model, epoch: 38500, test loss: 0.631100\n",
      "model, epoch: 38500, train loss: 0.606881\n",
      "time elapse: 141.941\n",
      "model, epoch: 39000, test loss: 0.631039\n",
      "model, epoch: 39000, train loss: 0.606287\n",
      "time elapse: 144.011\n",
      "model, epoch: 39500, test loss: 0.630987\n",
      "model, epoch: 39500, train loss: 0.605701\n",
      "time elapse: 146.107\n",
      "model, epoch: 40000, test loss: 0.630944\n",
      "model, epoch: 40000, train loss: 0.605124\n",
      "time elapse: 148.237\n",
      "model, epoch: 40500, test loss: 0.630909\n",
      "model, epoch: 40500, train loss: 0.604555\n",
      "time elapse: 150.568\n",
      "model, epoch: 41000, test loss: 0.630882\n",
      "model, epoch: 41000, train loss: 0.603994\n",
      "time elapse: 152.906\n",
      "model, epoch: 41500, test loss: 0.630863\n",
      "model, epoch: 41500, train loss: 0.603441\n",
      "time elapse: 155.356\n",
      "model, epoch: 42000, test loss: 0.630853\n",
      "model, epoch: 42000, train loss: 0.602897\n",
      "time elapse: 157.776\n",
      "model, epoch: 42500, test loss: 0.630853\n",
      "model, epoch: 42500, train loss: 0.602361\n",
      "time elapse: 160.006\n",
      "model, epoch: 43000, test loss: 0.630862\n",
      "model, epoch: 43000, train loss: 0.601833\n",
      "time elapse: 162.324\n",
      "model, epoch: 43500, test loss: 0.630878\n",
      "model, epoch: 43500, train loss: 0.601313\n",
      "time elapse: 164.606\n",
      "model, epoch: 44000, test loss: 0.630901\n",
      "model, epoch: 44000, train loss: 0.600801\n",
      "time elapse: 166.946\n",
      "model, epoch: 44500, test loss: 0.630930\n",
      "model, epoch: 44500, train loss: 0.600297\n",
      "time elapse: 169.400\n",
      "model, epoch: 45000, test loss: 0.630965\n",
      "model, epoch: 45000, train loss: 0.599800\n",
      "time elapse: 171.684\n",
      "model, epoch: 45500, test loss: 0.631007\n",
      "model, epoch: 45500, train loss: 0.599311\n",
      "time elapse: 174.212\n",
      "model, epoch: 46000, test loss: 0.631054\n",
      "model, epoch: 46000, train loss: 0.598830\n",
      "time elapse: 176.386\n",
      "model, epoch: 46500, test loss: 0.631109\n",
      "model, epoch: 46500, train loss: 0.598356\n",
      "time elapse: 178.848\n",
      "model, epoch: 47000, test loss: 0.631171\n",
      "model, epoch: 47000, train loss: 0.597889\n",
      "time elapse: 181.279\n",
      "model, epoch: 47500, test loss: 0.631239\n",
      "model, epoch: 47500, train loss: 0.597430\n",
      "time elapse: 183.499\n",
      "model, epoch: 48000, test loss: 0.631314\n",
      "model, epoch: 48000, train loss: 0.596979\n",
      "time elapse: 185.704\n",
      "model, epoch: 48500, test loss: 0.631395\n",
      "model, epoch: 48500, train loss: 0.596535\n",
      "time elapse: 188.041\n",
      "model, epoch: 49000, test loss: 0.631481\n",
      "model, epoch: 49000, train loss: 0.596098\n",
      "time elapse: 190.293\n",
      "model, epoch: 49500, test loss: 0.631573\n",
      "model, epoch: 49500, train loss: 0.595668\n",
      "time elapse: 192.860\n",
      "model, epoch: 50000, test loss: 0.631669\n",
      "model, epoch: 50000, train loss: 0.595246\n",
      "time elapse: 195.468\n",
      "model, epoch: 50500, test loss: 0.631770\n",
      "model, epoch: 50500, train loss: 0.594831\n",
      "time elapse: 197.953\n",
      "model, epoch: 51000, test loss: 0.631876\n",
      "model, epoch: 51000, train loss: 0.594422\n",
      "time elapse: 200.204\n",
      "model, epoch: 51500, test loss: 0.631987\n",
      "model, epoch: 51500, train loss: 0.594020\n",
      "time elapse: 202.455\n",
      "model, epoch: 52000, test loss: 0.632104\n",
      "model, epoch: 52000, train loss: 0.593625\n",
      "time elapse: 204.780\n",
      "model, epoch: 52500, test loss: 0.632228\n",
      "model, epoch: 52500, train loss: 0.593237\n",
      "time elapse: 207.181\n",
      "model, epoch: 53000, test loss: 0.632358\n",
      "model, epoch: 53000, train loss: 0.592854\n",
      "time elapse: 209.536\n",
      "model, epoch: 53500, test loss: 0.632494\n",
      "model, epoch: 53500, train loss: 0.592478\n",
      "time elapse: 211.954\n",
      "model, epoch: 54000, test loss: 0.632634\n",
      "model, epoch: 54000, train loss: 0.592108\n",
      "time elapse: 214.243\n",
      "model, epoch: 54500, test loss: 0.632776\n",
      "model, epoch: 54500, train loss: 0.591744\n",
      "time elapse: 216.586\n",
      "model, epoch: 55000, test loss: 0.632921\n",
      "model, epoch: 55000, train loss: 0.591386\n",
      "time elapse: 218.925\n",
      "model, epoch: 55500, test loss: 0.633069\n",
      "model, epoch: 55500, train loss: 0.591033\n",
      "time elapse: 221.283\n",
      "model, epoch: 56000, test loss: 0.633222\n",
      "model, epoch: 56000, train loss: 0.590685\n",
      "time elapse: 223.611\n",
      "model, epoch: 56500, test loss: 0.633380\n",
      "model, epoch: 56500, train loss: 0.590344\n",
      "time elapse: 226.070\n",
      "model, epoch: 57000, test loss: 0.633543\n",
      "model, epoch: 57000, train loss: 0.590007\n",
      "time elapse: 228.395\n",
      "model, epoch: 57500, test loss: 0.633711\n",
      "model, epoch: 57500, train loss: 0.589676\n",
      "time elapse: 230.795\n",
      "model, epoch: 58000, test loss: 0.633884\n",
      "model, epoch: 58000, train loss: 0.589350\n",
      "time elapse: 233.172\n",
      "model, epoch: 58500, test loss: 0.634061\n",
      "model, epoch: 58500, train loss: 0.589029\n",
      "time elapse: 235.707\n",
      "model, epoch: 59000, test loss: 0.634243\n",
      "model, epoch: 59000, train loss: 0.588713\n",
      "time elapse: 238.146\n",
      "model, epoch: 59500, test loss: 0.634429\n",
      "model, epoch: 59500, train loss: 0.588401\n",
      "time elapse: 240.645\n",
      "model, epoch: 60000, test loss: 0.634619\n",
      "model, epoch: 60000, train loss: 0.588095\n",
      "time elapse: 243.138\n",
      "model, epoch: 60500, test loss: 0.634811\n",
      "model, epoch: 60500, train loss: 0.587793\n",
      "time elapse: 245.664\n",
      "model, epoch: 61000, test loss: 0.635005\n",
      "model, epoch: 61000, train loss: 0.587496\n",
      "time elapse: 248.094\n",
      "model, epoch: 61500, test loss: 0.635201\n",
      "model, epoch: 61500, train loss: 0.587204\n",
      "time elapse: 250.706\n",
      "model, epoch: 62000, test loss: 0.635398\n",
      "model, epoch: 62000, train loss: 0.586916\n",
      "time elapse: 253.264\n",
      "model, epoch: 62500, test loss: 0.635594\n",
      "model, epoch: 62500, train loss: 0.586632\n",
      "time elapse: 255.799\n",
      "model, epoch: 63000, test loss: 0.635790\n",
      "model, epoch: 63000, train loss: 0.586352\n",
      "time elapse: 258.351\n",
      "model, epoch: 63500, test loss: 0.635982\n",
      "model, epoch: 63500, train loss: 0.586077\n",
      "time elapse: 260.974\n",
      "model, epoch: 64000, test loss: 0.636173\n",
      "model, epoch: 64000, train loss: 0.585805\n",
      "time elapse: 263.661\n",
      "model, epoch: 64500, test loss: 0.636363\n",
      "model, epoch: 64500, train loss: 0.585538\n",
      "time elapse: 266.264\n",
      "model, epoch: 65000, test loss: 0.636555\n",
      "model, epoch: 65000, train loss: 0.585274\n",
      "time elapse: 268.799\n",
      "model, epoch: 65500, test loss: 0.636750\n",
      "model, epoch: 65500, train loss: 0.585014\n",
      "time elapse: 271.416\n",
      "model, epoch: 66000, test loss: 0.636948\n",
      "model, epoch: 66000, train loss: 0.584757\n",
      "time elapse: 274.014\n",
      "model, epoch: 66500, test loss: 0.637146\n",
      "model, epoch: 66500, train loss: 0.584504\n",
      "time elapse: 276.588\n",
      "model, epoch: 67000, test loss: 0.637345\n",
      "model, epoch: 67000, train loss: 0.584254\n",
      "time elapse: 279.105\n",
      "model, epoch: 67500, test loss: 0.637544\n",
      "model, epoch: 67500, train loss: 0.584007\n",
      "time elapse: 281.703\n",
      "model, epoch: 68000, test loss: 0.637746\n",
      "model, epoch: 68000, train loss: 0.583764\n",
      "time elapse: 284.326\n",
      "model, epoch: 68500, test loss: 0.637950\n",
      "model, epoch: 68500, train loss: 0.583524\n",
      "time elapse: 287.001\n",
      "model, epoch: 69000, test loss: 0.638156\n",
      "model, epoch: 69000, train loss: 0.583286\n",
      "time elapse: 289.631\n",
      "model, epoch: 69500, test loss: 0.638364\n",
      "model, epoch: 69500, train loss: 0.583052\n",
      "time elapse: 292.319\n",
      "model, epoch: 70000, test loss: 0.638576\n",
      "model, epoch: 70000, train loss: 0.582821\n",
      "time elapse: 294.962\n",
      "model, epoch: 70500, test loss: 0.638791\n",
      "model, epoch: 70500, train loss: 0.582593\n",
      "time elapse: 297.594\n",
      "model, epoch: 71000, test loss: 0.639008\n",
      "model, epoch: 71000, train loss: 0.582367\n",
      "time elapse: 300.472\n",
      "model, epoch: 71500, test loss: 0.639228\n",
      "model, epoch: 71500, train loss: 0.582144\n",
      "time elapse: 303.223\n",
      "model, epoch: 72000, test loss: 0.639449\n",
      "model, epoch: 72000, train loss: 0.581924\n",
      "time elapse: 306.085\n",
      "model, epoch: 72500, test loss: 0.639671\n",
      "model, epoch: 72500, train loss: 0.581706\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time elapse: 308.884\n",
      "model, epoch: 73000, test loss: 0.639891\n",
      "model, epoch: 73000, train loss: 0.581491\n",
      "time elapse: 311.697\n",
      "model, epoch: 73500, test loss: 0.640110\n",
      "model, epoch: 73500, train loss: 0.581278\n",
      "time elapse: 314.422\n",
      "model, epoch: 74000, test loss: 0.640329\n",
      "model, epoch: 74000, train loss: 0.581067\n",
      "time elapse: 317.101\n",
      "model, epoch: 74500, test loss: 0.640548\n",
      "model, epoch: 74500, train loss: 0.580858\n",
      "time elapse: 319.816\n",
      "model, epoch: 75000, test loss: 0.640768\n",
      "model, epoch: 75000, train loss: 0.580652\n",
      "time elapse: 322.519\n",
      "model, epoch: 75500, test loss: 0.640985\n",
      "model, epoch: 75500, train loss: 0.580447\n",
      "time elapse: 325.252\n",
      "model, epoch: 76000, test loss: 0.641201\n",
      "model, epoch: 76000, train loss: 0.580245\n",
      "time elapse: 327.970\n",
      "model, epoch: 76500, test loss: 0.641414\n",
      "model, epoch: 76500, train loss: 0.580044\n",
      "time elapse: 330.772\n",
      "model, epoch: 77000, test loss: 0.641627\n",
      "model, epoch: 77000, train loss: 0.579845\n",
      "time elapse: 333.558\n",
      "model, epoch: 77500, test loss: 0.641838\n",
      "model, epoch: 77500, train loss: 0.579649\n",
      "time elapse: 336.476\n",
      "model, epoch: 78000, test loss: 0.642049\n",
      "model, epoch: 78000, train loss: 0.579453\n",
      "time elapse: 339.300\n",
      "model, epoch: 78500, test loss: 0.642258\n",
      "model, epoch: 78500, train loss: 0.579260\n",
      "time elapse: 342.075\n",
      "model, epoch: 79000, test loss: 0.642466\n",
      "model, epoch: 79000, train loss: 0.579069\n",
      "time elapse: 344.852\n",
      "model, epoch: 79500, test loss: 0.642673\n",
      "model, epoch: 79500, train loss: 0.578879\n",
      "time elapse: 347.712\n",
      "model, epoch: 80000, test loss: 0.642879\n",
      "model, epoch: 80000, train loss: 0.578690\n",
      "time elapse: 350.529\n",
      "model, epoch: 80500, test loss: 0.643083\n",
      "model, epoch: 80500, train loss: 0.578503\n",
      "time elapse: 353.335\n",
      "model, epoch: 81000, test loss: 0.643286\n",
      "model, epoch: 81000, train loss: 0.578318\n",
      "time elapse: 356.162\n",
      "model, epoch: 81500, test loss: 0.643488\n",
      "model, epoch: 81500, train loss: 0.578134\n",
      "time elapse: 358.987\n",
      "model, epoch: 82000, test loss: 0.643690\n",
      "model, epoch: 82000, train loss: 0.577951\n",
      "time elapse: 361.977\n",
      "model, epoch: 82500, test loss: 0.643890\n",
      "model, epoch: 82500, train loss: 0.577769\n",
      "time elapse: 364.806\n",
      "model, epoch: 83000, test loss: 0.644088\n",
      "model, epoch: 83000, train loss: 0.577589\n",
      "time elapse: 367.639\n",
      "model, epoch: 83500, test loss: 0.644285\n",
      "model, epoch: 83500, train loss: 0.577410\n",
      "time elapse: 370.508\n",
      "model, epoch: 84000, test loss: 0.644481\n",
      "model, epoch: 84000, train loss: 0.577231\n",
      "time elapse: 373.385\n",
      "model, epoch: 84500, test loss: 0.644677\n",
      "model, epoch: 84500, train loss: 0.577054\n",
      "time elapse: 376.393\n",
      "model, epoch: 85000, test loss: 0.644873\n",
      "model, epoch: 85000, train loss: 0.576878\n",
      "time elapse: 379.419\n",
      "model, epoch: 85500, test loss: 0.645068\n",
      "model, epoch: 85500, train loss: 0.576703\n",
      "time elapse: 382.304\n",
      "model, epoch: 86000, test loss: 0.645263\n",
      "model, epoch: 86000, train loss: 0.576529\n",
      "time elapse: 385.257\n",
      "model, epoch: 86500, test loss: 0.645456\n",
      "model, epoch: 86500, train loss: 0.576356\n",
      "time elapse: 388.151\n",
      "model, epoch: 87000, test loss: 0.645649\n",
      "model, epoch: 87000, train loss: 0.576184\n",
      "time elapse: 391.139\n",
      "model, epoch: 87500, test loss: 0.645841\n",
      "model, epoch: 87500, train loss: 0.576013\n",
      "time elapse: 394.189\n",
      "model, epoch: 88000, test loss: 0.646031\n",
      "model, epoch: 88000, train loss: 0.575843\n",
      "time elapse: 397.025\n",
      "model, epoch: 88500, test loss: 0.646219\n",
      "model, epoch: 88500, train loss: 0.575674\n",
      "time elapse: 400.002\n",
      "model, epoch: 89000, test loss: 0.646406\n",
      "model, epoch: 89000, train loss: 0.575505\n",
      "time elapse: 402.943\n",
      "model, epoch: 89500, test loss: 0.646590\n",
      "model, epoch: 89500, train loss: 0.575337\n",
      "time elapse: 405.903\n",
      "model, epoch: 90000, test loss: 0.646771\n",
      "model, epoch: 90000, train loss: 0.575170\n",
      "time elapse: 408.902\n",
      "model, epoch: 90500, test loss: 0.646949\n",
      "model, epoch: 90500, train loss: 0.575004\n",
      "time elapse: 411.958\n",
      "model, epoch: 91000, test loss: 0.647124\n",
      "model, epoch: 91000, train loss: 0.574838\n",
      "time elapse: 415.006\n",
      "model, epoch: 91500, test loss: 0.647296\n",
      "model, epoch: 91500, train loss: 0.574673\n",
      "time elapse: 418.203\n",
      "model, epoch: 92000, test loss: 0.647466\n",
      "model, epoch: 92000, train loss: 0.574509\n",
      "time elapse: 421.251\n",
      "model, epoch: 92500, test loss: 0.647633\n",
      "model, epoch: 92500, train loss: 0.574345\n",
      "time elapse: 424.441\n",
      "model, epoch: 93000, test loss: 0.647799\n",
      "model, epoch: 93000, train loss: 0.574181\n",
      "time elapse: 427.466\n",
      "model, epoch: 93500, test loss: 0.647963\n",
      "model, epoch: 93500, train loss: 0.574018\n",
      "time elapse: 430.464\n",
      "model, epoch: 94000, test loss: 0.648127\n",
      "model, epoch: 94000, train loss: 0.573856\n",
      "time elapse: 433.509\n",
      "model, epoch: 94500, test loss: 0.648289\n",
      "model, epoch: 94500, train loss: 0.573694\n",
      "time elapse: 436.561\n",
      "model, epoch: 95000, test loss: 0.648451\n",
      "model, epoch: 95000, train loss: 0.573532\n",
      "time elapse: 439.617\n",
      "model, epoch: 95500, test loss: 0.648613\n",
      "model, epoch: 95500, train loss: 0.573371\n",
      "time elapse: 442.669\n",
      "model, epoch: 96000, test loss: 0.648776\n",
      "model, epoch: 96000, train loss: 0.573210\n",
      "time elapse: 445.715\n",
      "model, epoch: 96500, test loss: 0.648938\n",
      "model, epoch: 96500, train loss: 0.573050\n",
      "time elapse: 448.915\n",
      "model, epoch: 97000, test loss: 0.649099\n",
      "model, epoch: 97000, train loss: 0.572890\n",
      "time elapse: 452.004\n",
      "model, epoch: 97500, test loss: 0.649260\n",
      "model, epoch: 97500, train loss: 0.572730\n",
      "time elapse: 455.080\n",
      "model, epoch: 98000, test loss: 0.649420\n",
      "model, epoch: 98000, train loss: 0.572571\n",
      "time elapse: 458.187\n",
      "model, epoch: 98500, test loss: 0.649580\n",
      "model, epoch: 98500, train loss: 0.572411\n",
      "time elapse: 461.512\n",
      "model, epoch: 99000, test loss: 0.649740\n",
      "model, epoch: 99000, train loss: 0.572253\n",
      "time elapse: 464.741\n",
      "model, epoch: 99500, test loss: 0.649900\n",
      "model, epoch: 99500, train loss: 0.572094\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import matplotlib\n",
    "matplotlib.use('Agg')   \n",
    "import pickle\n",
    "import time  \n",
    "import shutil \n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt   \n",
    "import platform\n",
    "#from BasicFunc import mySaveFig, mkdir\n",
    "from mpl_toolkits.mplot3d import Axes3D \n",
    "from matplotlib import cm\n",
    "from matplotlib.ticker import LinearLocator, FormatStrFormatter\n",
    "print('hello,world')\n",
    "#Plot Parameters\n",
    "Leftp=0.18\n",
    "Bottomp=0.18\n",
    "Widthp=0.88-Leftp\n",
    "Heightp=0.9-Bottomp\n",
    "pos=[Leftp,Bottomp,Widthp,Heightp]\n",
    "\n",
    "def mkdir(fn):\n",
    "    if not os.path.isdir(fn):\n",
    "        os.mkdir(fn)\n",
    "def mySaveFig(pltm, fntmp,fp=0,ax=0,isax=0,iseps=0,isShowPic=0):\n",
    "    if isax==1:\n",
    "        #pltm.legend(fontsize=18)\n",
    "        # plt.title(y_name,fontsize=14)\n",
    "#        ax.set_xlabel('step',fontsize=18)\n",
    "#        ax.set_ylabel('loss',fontsize=18)\n",
    "        pltm.rc('xtick',labelsize=18)\n",
    "        pltm.rc('ytick',labelsize=18)\n",
    "        ax.set_position(pos, which='both')\n",
    "    fnm='%s.png'%(fntmp)\n",
    "    pltm.savefig(fnm)\n",
    "    if iseps:\n",
    "        fnm='%s.eps'%(fntmp)\n",
    "        pltm.savefig(fnm, format='eps', dpi=600)\n",
    "    if fp!=0:\n",
    "        fp.savefig(\"%s.pdf\"%(fntmp), bbox_inches='tight')\n",
    "    if isShowPic==1:\n",
    "        pltm.show() \n",
    "    elif isShowPic==-1:\n",
    "        return\n",
    "    else:\n",
    "        pltm.close()\n",
    "        \n",
    "R_variable={}  ### used for saved all parameters and data\n",
    "### mkdir a folder to save all output\n",
    "BaseDir = 'fitnd/'\n",
    "subFolderName = '%s'%(int(np.absolute(np.random.normal([1])*100000))//int(1)) \n",
    "FolderName = '%s%s/'%(BaseDir,subFolderName)\n",
    "mkdir(BaseDir)\n",
    "mkdir(FolderName)\n",
    "mkdir('%smodel/'%(FolderName))\n",
    "R_variable['FolderName']=FolderName \n",
    "if  not platform.system()=='Windows':\n",
    "    shutil.copy(__file__,'%s%s'%(FolderName,os.path.basename(__file__)))    \n",
    "R_variable['input_dim']=5\n",
    "R_variable['epsion']=0.1\n",
    "\n",
    "#***Function we need to fit***\n",
    "def get_y_func(xs):\n",
    "    tmp=0\n",
    "    print(\"input x_data is: \",xs)\n",
    "    for ii in range(R_variable['input_dim']):\n",
    "        #tmp+=np.cos(4*xs[:,ii:ii+1])                    #***elementwise calculate cos value***\n",
    "        #tmp+=np.cos(10*xs[:,ii:ii+1])+np.cos(50*xs[:,ii:ii+1])+np.cos(200*xs[:,ii:ii+1])\n",
    "        tmp += np.sin(50*xs[:,ii:ii+1])*np.cos(50*xs[:,ii:ii+1])\n",
    "    return tmp\n",
    "# 若有n个输入， f(x1,x2,...,xn)=∑(cos(4*xi))\n",
    "\n",
    "R_variable['output_dim']=1\n",
    "R_variable['ActFuc']=1   ###  0: ReLU; 1: Tanh; 3:sin;4: x**5,, 5: sigmoid  6 sigmoid derivate\n",
    "#R_variable['hidden_units']=[1500,1500,1500]\n",
    "#R_variable['hidden_units']=[500,500,500]\n",
    "#R_variable['hidden_units']=[200,200,200]\n",
    "#R_variable['hidden_units']=[300,300,300,300]\n",
    "R_variable['hidden_units']=[200]\n",
    "### initialization standard deviation\n",
    "R_variable['astddev']=np.sqrt(1/20) # for weight\n",
    "R_variable['bstddev']=np.sqrt(1/20)# for bias terms2\n",
    "R_variable['ASI']=0\n",
    "R_variable['learning_rate']=1e-6\n",
    "R_variable['learning_rateDecay']=2e-7\n",
    "R_variable['seed']=0\n",
    "### setup for activation function\n",
    "#***The parameters used to train the model***\n",
    "plotepoch=500\n",
    "R_variable['train_size']=1000;                    ### training size\n",
    "R_variable['batch_size']=R_variable['train_size'] # int(np.floor(R_variable['train_size'])) ### batch size\n",
    "R_variable['test_size']=R_variable['train_size']  ### test size\n",
    "R_variable['x_start']=-np.pi/2  #math.pi*3 ### start point of input\n",
    "R_variable['x_end']=np.pi/2  #6.28/4 #math.pi*3  ### end point of input\n",
    "R_variable['tol']=-1e10\n",
    "R_variable['Total_Step']=600000  ### the training step. Set a big number, if it converges, can manually stop training \n",
    "R_variable['FolderName']=FolderName   ### folder for save images\n",
    "# 上述均为训练模型的参数\n",
    "print(R_variable) \n",
    "\n",
    "#***Training and test data are generated here***\n",
    "# 这里生成训练、测试数据。输入为一维的话，应该是均匀地在区间里线性分布；多维的话，每一个数据均为[start, end]中的随机数\n",
    "if R_variable['input_dim']==1:\n",
    "    R_variable['test_inputs'] =np.reshape(np.linspace(R_variable['x_start'], R_variable['x_end'], \n",
    "                                                      num=R_variable['test_size'],\n",
    "                                                      endpoint=True),[R_variable['test_size'],1])\n",
    "    #n_size=R_variable['train_size']\n",
    "    R_variable['train_inputs']=np.reshape(np.linspace(R_variable['x_start'], R_variable['x_end'],\n",
    "                                                      num=R_variable['train_size'],endpoint=True),\n",
    "                                          [R_variable['train_size'],1])\n",
    "else:\n",
    "    R_variable['test_inputs']=np.random.rand(R_variable['test_size'],R_variable['input_dim'])\\\n",
    "    *(R_variable['x_end']-R_variable['x_start'])+R_variable['x_start']\n",
    "    R_variable['train_inputs']=np.random.rand(R_variable['train_size'],R_variable['input_dim'])\\\n",
    "    *(R_variable['x_end']-R_variable['x_start'])+R_variable['x_start']\n",
    "\n",
    "\n",
    "\n",
    "test_inputs=R_variable['test_inputs']\n",
    "train_inputs=R_variable['train_inputs']\n",
    "R_variable['y_true_test']= get_y_func(test_inputs)\n",
    "R_variable['y_true_train']=get_y_func(train_inputs)\n",
    "\n",
    "#plt.figure()\n",
    "#plt.plot(test_inputs,R_variable['y_true_test'])\n",
    "\n",
    "t0=time.time() \n",
    "\n",
    "def add_layer2(x,input_dim = 1,output_dim = 1,isresnet=0,astddev=0.05,\n",
    "               bstddev=0.05,ActFuc=0,seed=0, name_scope='hidden'):\n",
    "    if not seed==0:\n",
    "        tf.set_random_seed(seed)\n",
    "    \n",
    "    with tf.variable_scope(name_scope, reuse=tf.AUTO_REUSE):\n",
    "        ua_w = tf.get_variable(name='ua_w', initializer=astddev)\n",
    "        ua_b = tf.get_variable(name='ua_b', initializer=bstddev) \n",
    "        z=tf.matmul(x, ua_w) + ua_b\n",
    "        \n",
    "        \n",
    "        if ActFuc==1:\n",
    "            output_z = tf.nn.tanh(z)\n",
    "            print('tanh')\n",
    "        elif ActFuc==3:\n",
    "            output_z = tf.sin(z)\n",
    "            print('sin')\n",
    "        elif ActFuc==0:\n",
    "            output_z = tf.nn.relu(z)\n",
    "            print('relu')\n",
    "        elif ActFuc==4:\n",
    "            output_z = z**50\n",
    "            print('z**50')\n",
    "        elif ActFuc==5:\n",
    "            output_z = tf.nn.sigmoid(z)\n",
    "            print('sigmoid')\n",
    "            \n",
    "        L2Wight= tf.nn.l2_loss(ua_w) \n",
    "        if isresnet and input_dim==output_dim:\n",
    "            output_z=output_z+x\n",
    "        return output_z,ua_w,ua_b,L2Wight\n",
    "\n",
    "def getWini(hidden_units=[10,20,40],input_dim = 1,output_dim_final = 1,astddev=0.05,bstddev=0.05):\n",
    "    \n",
    "    hidden_num = len(hidden_units)\n",
    "    #print(hidden_num)\n",
    "    add_hidden = [input_dim] + hidden_units;\n",
    "    # 加入第一层的输入维数（input_dim）\n",
    "    w_Univ0=[]\n",
    "    b_Univ0=[]\n",
    "    #***Set weight and biase of each neuron***\n",
    "    for i in range(hidden_num):\n",
    "        input_dim = add_hidden[i]\n",
    "        output_dim = add_hidden[i+1]\n",
    "        ua_w=np.float32(np.random.normal(loc=0.0,scale=astddev,size=[input_dim,output_dim]))\n",
    "        ua_b=np.float32(np.random.normal(loc=0.0,scale=bstddev,size=[output_dim]))\n",
    "        w_Univ0.append(ua_w)\n",
    "        b_Univ0.append(ua_b)\n",
    "        # 构造每一层，，output = input * ua_w + ua_b\n",
    "    #***Set weight and biase of the last layer, they are random value***    \n",
    "    ua_w=np.float32(np.random.normal(loc=0.0,scale=astddev,size=[hidden_units[hidden_num-1], output_dim_final]))\n",
    "    ua_b=np.float32(np.random.normal(loc=0.0,scale=bstddev,size=[output_dim_final]))\n",
    "    w_Univ0.append(ua_w)\n",
    "    b_Univ0.append(ua_b)\n",
    "    # 同样构造最后一层，输出为output_dim_final，将矩阵初始值加入（随机值）\n",
    "    return w_Univ0, b_Univ0\n",
    "\n",
    "def univAprox2(x0, hidden_units=[10,20,40],input_dim = 1,output_dim_final = 1,\n",
    "               isresnet=0,astddev=0.05,bstddev=0.05,\n",
    "               ActFuc=0,seed=0,ASI=1):\n",
    "    if seed==0:\n",
    "        seed=time.time()\n",
    "    # The simple case is f: R -> R \n",
    "    hidden_num = len(hidden_units)\n",
    "    #print(hidden_num)\n",
    "    add_hidden = [input_dim] + hidden_units;\n",
    "    print(\"add_hidden:\",add_hidden)\n",
    "    w_Univ=[]\n",
    "    b_Univ=[] \n",
    "    L2w_all=0\n",
    "    w_Univ0, b_Univ0=getWini(hidden_units=hidden_units,input_dim = input_dim,\n",
    "                             output_dim_final= output_dim_final,astddev=astddev,bstddev=bstddev)\n",
    "    output=x0\n",
    "    for i in range(hidden_num):\n",
    "        input_dim = add_hidden[i]\n",
    "        output_dim = add_hidden[i+1]\n",
    "        print('input_dim:%s, output_dim:%s'%(input_dim,output_dim))\n",
    "        name_scope = 'hidden' + np.str(i+1)   \n",
    "        output,ua_w,ua_b,L2Wight_tmp=add_layer2(output,input_dim,output_dim,isresnet=isresnet,\n",
    "                                               astddev=w_Univ0[i],bstddev=b_Univ0[i], ActFuc=ActFuc,\n",
    "                                              seed=seed, name_scope= name_scope)\n",
    "        w_Univ.append(ua_w)\n",
    "        b_Univ.append(ua_b)\n",
    "        L2w_all = L2w_all + L2Wight_tmp\n",
    "        # 这个函数会将output进行一次非线性的activation，同时获得这一层的矩阵参数ua_w, ua_b\n",
    "        # 应该是output = activation(input) * ua_w + ua_b\n",
    "        # L2Weight_tmp计算∑(ua_w)^2，应该是L2正则化    \n",
    "        \n",
    "    ua_we = tf.get_variable(\n",
    "            name='ua_we'\n",
    "            #, shape=[hidden_units[hidden_num-1], output_dim_final]\n",
    "            , initializer=w_Univ0[-1]\n",
    "        )\n",
    "    ua_be = tf.get_variable(\n",
    "            name='ua_be'\n",
    "            #, shape=[1,output_dim_final]\n",
    "            , initializer=b_Univ0[-1]\n",
    "        )\n",
    "    \n",
    "    z1 = tf.matmul(output, ua_we)+ua_be\n",
    "    w_Univ.append(ua_we)\n",
    "    b_Univ.append(ua_be)\n",
    "    # 同样，最后一层output = input * ua_we + ua_be，但我不知道为啥要把ua_we,ua_be的shape注释掉\n",
    "    \n",
    "    # you can ignore this trick for now. Consider ASI=False\n",
    "    if ASI:\n",
    "        output=x0\n",
    "        for i in range(hidden_num):\n",
    "            input_dim = add_hidden[i]\n",
    "            output_dim = add_hidden[i+1]\n",
    "            print('input_dim:%s, output_dim:%s'%(input_dim,output_dim))\n",
    "            name_scope = 'hidden' + np.str(i+1+hidden_num) 
    "            output,ua_w,ua_b,L2Wight_tmp=add_layer2(output,input_dim,output_dim,isresnet=isresnet,\n",
    "                                               astddev=w_Univ0[i],bstddev=b_Univ0[i], ActFuc=ActFuc,\n",
    "                                               seed=seed, name_scope= name_scope)\n",
    "            \n",
    "            \n",
    "                                               \n",
    "        ua_we = tf.get_variable(\n",
    "                name='ua_wei2'\n",
    "                #, shape=[hidden_units[hidden_num-1], output_dim_final]\n",
    "                , initializer=-w_Univ0[-1]\n",
    "            )\n",
    "        ua_be = tf.get_variable(\n",
    "                name='ua_bei2'\n",
    "                #, shape=[1,output_dim_final]\n",
    "                , initializer=-b_Univ0[-1]\n",
    "            )\n",
    "        z2 = tf.matmul(output, ua_we)+ua_be\n",
    "    else:\n",
    "        z2=0\n",
    "    z=z1+z2\n",
    "    return z,w_Univ,b_Univ,L2w_all\n",
    "    # z是预测结果,w_Univ,b_Univ是计算过程中用到的参数，L2w_all对应w_Univ每一层的L2的值\n",
    "\n",
    "tf.reset_default_graph()\n",
    "with tf.variable_scope('Graph',reuse=tf.AUTO_REUSE) as scope:\n",
    "    # Our inputs will be a batch of values taken by our functions\n",
    "    x = tf.placeholder(tf.float32, shape=[None, R_variable['input_dim']], name=\"x\")\n",
    "    \n",
    "    \n",
    "    y_true = tf.placeholder_with_default(input=[[0.0]], shape=[None, R_variable['output_dim']], name=\"y\")\n",
    "    in_learning_rate= tf.placeholder_with_default(input=1e-3,shape=[],name='lr')\n",
    "    y,_,_,_ = univAprox2(x, R_variable['hidden_units'],input_dim = R_variable['input_dim'],\n",
    "                                            astddev=R_variable['astddev'],bstddev=R_variable['bstddev'],\n",
    "                                            ActFuc=R_variable['ActFuc'],\n",
    "                                            seed=R_variable['seed'],ASI=R_variable['ASI'])\n",
    "                                            # 这里在计算训练值\n",
    "    \n",
    "    loss=tf.reduce_mean(tf.square(y_true-y))\n",
    "    # We define our train operation using the Adam optimizer\n",
    "    adam = tf.compat.v1.train.AdamOptimizer(learning_rate=in_learning_rate) \n"
    "    train_op = adam.minimize(loss)\n",
    "    # 这边应该是在更新矩阵的参数\n",
    "\n",
    "#使用GPU的过程中，根据需要占用内存"
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth=True  \n",
    "sess = tf.Session(config=config)\n",
    "sess.run(tf.global_variables_initializer())\n",
    "saver = tf.train.Saver()  \n",
    "        \n",
    "class model():\n",
    "    def __init__(self): \n",
    "        R_variable['y_train']=[]\n",
    "        R_variable['y_test']=[]\n",
    "        R_variable['loss_test']=[]\n",
    "        R_variable['loss_train']=[]\n",
    "        # 记录训练结果\n",
    "    \n",
    "        nametmp='%smodel/'%(FolderName)\n",
    "        mkdir(nametmp)\n",
    "        w_Univ0, b_Univ0=getWini(hidden_units=R_variable['hidden_units'],\n",
    "                                 input_dim = R_variable['input_dim'],\n",
    "                                 output_dim_final = R_variable['output_dim'],\n",
    "                                 astddev=R_variable['astddev'],\n",
    "                                 bstddev=R_variable['bstddev'])\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        saver.save(sess, \"%smodel.ckpt\"%(nametmp))\n",
    "        y_test, loss_test_tmp= sess.run([y,loss],feed_dict={x: test_inputs, y_true: R_variable['y_true_test']})\n",
    "        y_train,loss_train_tmp= sess.run([y,loss],feed_dict={x: train_inputs, y_true: R_variable['y_true_train']})\n",
    "        R_variable['y_train']=y_train\n",
    "        R_variable['y_test']=y_test\n",
    "        R_variable['loss_test'].append(loss_test_tmp)\n",
    "        R_variable['loss_train'].append(loss_train_tmp)\n",
    "        \n",
    "        self.ploty(name='ini') \n",
    "        # 初始化，读入数据\n",
    "        \n",
    "    def run_onestep(self):\n",
    "        y_test, loss_test_tmp= sess.run([y,loss],feed_dict={x: test_inputs, y_true: R_variable['y_true_test']})\n",
    "        \n",
    "        y_train,loss_train_tmp= sess.run([y,loss],feed_dict={x: train_inputs, y_true: R_variable['y_true_train']})\n",
    "            \n",
    "        if R_variable['train_size']>R_variable['batch_size']:\n",
    "            indperm = np.random.permutation(R_variable['train_size'])\n",
    "            nrun_epoch=np.int32(R_variable['train_size']/R_variable['batch_size'])\n",
    "            \n",
    "            for ijn in range(nrun_epoch):\n",
    "                ind = indperm[ijn*R_variable['batch_size']:(ijn+1)*R_variable['batch_size']] \n",
    "                _= sess.run(train_op, feed_dict={x: train_inputs[ind], y_true: R_variable['y_true_train'][ind],\n",
    "                                                  in_learning_rate:R_variable['learning_rate']})\n",
    "        else:\n",
    "            _ = sess.run(train_op, feed_dict={x: train_inputs, y_true: R_variable['y_true_train'],\n",
    "                                                  in_learning_rate:R_variable['learning_rate']})\n",
    "        R_variable['learning_rate']=R_variable['learning_rate']*(1-R_variable['learning_rateDecay'])\n",
    "            \n",
    "        R_variable['y_train']=y_train\n",
    "        R_variable['y_test']=y_test\n",
    "        R_variable['loss_test'].append(loss_test_tmp)\n",
    "        R_variable['loss_train'].append(loss_train_tmp)\n",
    "        # 运行一步，并记录\n",
    "        \n",
    "    def run(self,step_n=1):\n",
    "        nametmp='%smodel/model.ckpt'%(FolderName)\n",
    "        saver.restore(sess, nametmp)\n",
    "        # 每step_n步保存一次\n",
    "        for ii in range(step_n):\n",
    "            self.run_onestep()\n",
    "            if R_variable['loss_train'][-1]<R_variable['tol']:\n",
    "                # 这个错误检测,预防loss太大，浮点数爆炸，符号位被占用导致loss变成负数\n",
    "                print('model end, error:%s'%(R_variable['lossu_train'][-1]))\n",
    "                self.plotloss()\n",
    "                self.ploty()\n",
    "                self.savefile()\n",
    "                nametmp='%smodel/'%(FolderName)\n",
    "                shutil.rmtree(nametmp)\n",
    "                saver.save(sess, \"%smodel.ckpt\"%(nametmp))\n",
    "                break\n",
    "            if ii==0:\n",
    "                print('initial %s'%(np.max(R_variable['y_train'])))\n",
    "                \n",
    "            if ii%plotepoch==0:\n",
    "                print('time elapse: %.3f'%(time.time()-t0))\n",
    "                print('model, epoch: %d, test loss: %f' % (ii,R_variable['loss_test'][-1]))\n",
    "                print('model, epoch: %d, train loss: %f' % (ii,R_variable['loss_train'][-1]))\n",
    "                self.plotloss()\n",
    "                self.ploty(name='%s'%(ii))\n",
    "                self.savefile()\n",
    "                nametmp='%smodel/'%(FolderName)\n",
    "                shutil.rmtree(nametmp)\n",
    "                saver.save(sess, \"%smodel.ckpt\"%(nametmp))\n",
    "                # 画图记录\n",
    "            \n",
    "                \n",
    "    def plotloss(self):\n",
    "        plt.figure()\n",
    "        ax = plt.gca()\n",
    "        y1 = R_variable['loss_test']\n",
    "        y2 = R_variable['loss_train']\n",
    "        plt.plot(y1,'ro',label='Test')\n",
    "        plt.plot(y2,'g*',label='Train')\n",
    "        ax.set_xscale('log')\n",
    "        ax.set_yscale('log')                \n",
    "        plt.legend(fontsize=18)\n",
    "        plt.title('loss',fontsize=15)\n",
    "        plt.xlabel('epoch',fontsize=15)\n",
    "        fntmp = '%sloss'%(FolderName)\n",
    "        mySaveFig(plt,fntmp,ax=ax,isax=1,iseps=0)\n",
    "        \n",
    "            \n",
    "    def ploty(self,name=''):\n",
    "        \n",
    "        if R_variable['input_dim']==2:\n",
    "            # Make data.\n",
    "            X = np.arange(R_variable['x_start'], R_variable['x_end'], 0.1)\n",
    "            Y = np.arange(R_variable['x_start'], R_variable['x_end'], 0.1)\n",
    "            X, Y = np.meshgrid(X, Y)\n",
    "            xy=np.concatenate((np.reshape(X,[-1,1]),np.reshape(Y,[-1,1])),axis=1)\n",
    "            Z = np.reshape(get_y_func(xy),[len(X),-1])\n",
    "            fp = plt.figure()\n",
    "            ax = fp.gca(projection='3d')\n",
    "            # Plot the surface.\n",
    "            surf = ax.plot_surface(X, Y, Z-np.min(Z), cmap=cm.coolwarm,linewidth=0, antialiased=False)\n",
    "            # Customize the z axis.\n",
    "            #ax.set_zlim(-2.01, 2.01)\n",
    "            ax.zaxis.set_major_locator(LinearLocator(5))\n",
    "            ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))\n",
    "            # Add a color bar which maps values to colors.\n",
    "            fp.colorbar(surf, shrink=0.5, aspect=5)\n",
    "            ax.scatter(train_inputs[:,0], train_inputs[:,1], R_variable['y_train']-np.min(R_variable['y_train']))\n",
    "            fntmp = '%s2du%s'%(FolderName,name)\n",
    "            mySaveFig(plt,fntmp,ax=ax,isax=1,iseps=0)\n",
    "        if R_variable['input_dim']==1:\n",
    "            plt.figure()\n",
    "            ax = plt.gca()\n",
    "            y1 = R_variable['y_test']\n",
    "            y2 = R_variable['y_train']\n",
    "            y3 = R_variable['y_true_test']\n",
    "            plt.plot(test_inputs,y1,'ro',label='Test')\n",
    "            plt.plot(train_inputs,y2,'g*',label='Train')\n",
    "            plt.plot(test_inputs,y3,'b*',label='True')\n",
    "            plt.title('g2u',fontsize=15)        \n",
    "            plt.legend(fontsize=18) \n",
    "            fntmp = '%su_m%s'%(FolderName,name)\n",
    "            mySaveFig(plt,fntmp,ax=ax,isax=1,iseps=0)\n",
    "            \n",
    "    def savefile(self):\n",
    "        with open('%s/objs.pkl'%(FolderName), 'wb') as f:  # Python 3: open(..., 'wb')\n",
    "            pickle.dump(R_variable, f, protocol=4)\n",
    "         \n",
    "        text_file = open(\"%s/Output.txt\"%(FolderName), \"w\")\n",
    "        for para in R_variable:\n",
    "            if np.size(R_variable[para])>20:\n",
    "                continue\n",
    "            text_file.write('%s: %s\\n'%(para,R_variable[para]))\n",
    "        \n",
    "        text_file.close()\n",
    "        \n",
    "                \n",
    "            \n",
    "model1=model()\n",
    "model1.run(100000)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}