Fixed timestamps, line numbers and coordinates format.

davidcampos · Apr 2, 2019 · 36e203d · 36e203d
1 parent b918452
commit 36e203d
Showing 1 changed file with 37 additions and 27 deletions.
diff --git a/get_IOT_data.ipynb b/get_IOT_data.ipynb
@@ -377,7 +377,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 58,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -393,10 +393,11 @@
     "    for i in range(0,len(j['data']['attributes'])):\n",
     "        df[j['data']['attributes'][i]['attrName']] = j['data']['attributes'][i]['values']\n",
     "    ## expand location attributes\n",
-    "    for key in df['location'][0].keys():\n",
-    "        df[key] = str(df['location'][0][key])\n",
+    "    #for key in df['location'][0].keys():\n",
+    "    #    df[key] = str(df['location'][0][key])\n",
+    "    df['long'] = df['location'][0]['coordinates'][0]\n",
+    "    df['lat'] = df['location'][0]['coordinates'][1]\n",
     "    df['device_id'] = j['data']['entityId']\n",
-    "    df['indexT'] = j['data']['index']\n",
     "    #print(df)\n",
     "    return(df)\n",
     "    \n",
@@ -415,7 +416,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 59,
    "metadata": {},
    "outputs": [
     {
@@ -456,7 +457,34 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 61,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# DF operations\n",
+    "\n",
+    "import  time\n",
+    "import datetime\n",
+    "\n",
+    "# Convert timestamp to unixtime\n",
+    "def convert_df_to_unix(s):\n",
+    "    time_mask = \"%Y-%m-%dT%H:%M:%S.%f\"\n",
+    "    return(time.mktime(datetime.datetime.strptime(s, time_mask).timetuple()))\n",
+    " \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 62,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df['time'] = list(map(convert_df_to_unix, df['dateObserved']))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 65,
    "metadata": {},
    "outputs": [
     {
@@ -475,32 +503,14 @@
    ],
    "source": [
     "## Save columns to csv in a custom order\n",
-    "mandatory_column_list = [\"dateObserved\", \"coordinates\"]\n",
+    "mandatory_column_list = [\"time\", \"lat\", \"long\"]\n",
     "data_column_list = [\"CO\", \"NO2\", \"O3\", \"Ox\", \"PM1\", \"PM10\", \"PM25\"]\n",
     "for data_column in data_column_list:\n",
     "    filepath = \"./output/\" + \"IOT_\" + \"AirQuality_\" + data_column + \".csv\"\n",
     "    print(filepath)\n",
-    "    df.to_csv(filepath, columns = mandatory_column_list + [data_column] )"
+    "    df.to_csv(filepath, columns = mandatory_column_list + [data_column], \n",
+    "              index = False, na_rep = \"\", header = True)"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 32,
-   "metadata": {},
-   "outputs": [
-    {
-     "ename": "NameError",
-     "evalue": "name 'j' is not defined",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
-      "\u001b[0;32m<ipython-input-32-325732b9e9c9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
-      "\u001b[0;31mNameError\u001b[0m: name 'j' is not defined"
-     ]
-    }
-   ],
-   "source": []
   }
  ],
  "metadata": {