0_PlanetDownload.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aec5f4d4-daaf-4704-af0a-cf52e3756bfc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# load dependencies\n",
    "import os\n",
    "import requests\n",
    "import geopandas as gpd\n",
    "from requests.auth import HTTPBasicAuth\n",
    "from shapely.geometry import shape\n",
    "import numpy as np\n",
    "import subprocess\n",
    "import json\n",
    "import pathlib\n",
    "import pandas as pd\n",
    "headers = {'Content-Type': 'application/json'}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8c99f8d-a585-4b57-bd64-83ea8704e984",
   "metadata": {},
   "source": [
    "## SET USER DEFINITIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82d0722c-2397-4758-b6b0-27f28b9707b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "domainID = 'DAN'\n",
    "\n",
    "# enter the Planet user API key\n",
    "apiKey = ''\n",
    "# planet download specifications\n",
    "item_type = \"PSScene\"\n",
    "asset_type = \"ortho_analytic_4b_sr\"\n",
    "bundle_type = \"analytic_sr_udm2\"\n",
    "\n",
    "# data download location\n",
    "out_direc = '/Users/jpflug/Documents/Projects/cubesatReanaly/Data/Meadows/'+domainID\n",
    "\n",
    "# specify the download extents by WGS84 coordinates\n",
    "geo_json_geometry = {\n",
    "  \"type\": \"Polygon\",\n",
    "  \"coordinates\": [\n",
    "    [\n",
    "      [\n",
    "        -119.26174870015272,\n",
    "        37.91103872262529\n",
    "      ],\n",
    "      [\n",
    "        -119.26174870015272,\n",
    "        37.88853458822983\n",
    "      ],\n",
    "      [\n",
    "        -119.24988470101171,\n",
    "        37.88853458822983\n",
    "      ],\n",
    "      [\n",
    "        -119.24988470101171,\n",
    "        37.91103872262529\n",
    "      ],\n",
    "      [\n",
    "        -119.26174870015272,\n",
    "        37.91103872262529\n",
    "      ]\n",
    "    ]\n",
    "  ]\n",
    "}\n",
    "\n",
    "# filter images acquired in a certain date range\n",
    "date_range_filter = {\n",
    "  \"type\": \"DateRangeFilter\",\n",
    "  \"field_name\": \"acquired\",\n",
    "  \"config\": {\n",
    "    \"gte\": \"2020-01-01T00:00:00.000Z\",\n",
    "    \"lte\": \"2020-09-01T00:00:00.000Z\"\n",
    "  }\n",
    "}\n",
    "\n",
    "# filter any images based on a cloud cover filter\n",
    "cloud_cover_filter = {\n",
    "  \"type\": \"RangeFilter\",\n",
    "  \"field_name\": \"cloud_cover\",\n",
    "  \"config\": {\n",
    "    \"lte\": 0.05\n",
    "  }\n",
    "}\n",
    "\n",
    "# filter for items the overlap with our chosen geometry from above\n",
    "geometry_filter = {\n",
    "  \"type\": \"GeometryFilter\",\n",
    "  \"field_name\": \"geometry\",\n",
    "  \"config\": geo_json_geometry\n",
    "}\n",
    "\n",
    "# create a filter that combines our geo and date filters\n",
    "# could also use an \"OrFilter\"\n",
    "domain = {\n",
    "  \"type\": \"AndFilter\",\n",
    "  \"config\": [geometry_filter, date_range_filter, cloud_cover_filter]\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2292b2a4-3c64-4745-913d-92529fc820a5",
   "metadata": {},
   "source": [
    "## FUNCTIONS -- DO NOT EDIT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3bf8306b-7530-4ed1-8d8e-991b2711c2a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# functions based on PS API tools --> https://developers.planet.com/docs/apis/\n",
    "def build_payload(item_ids, item_type, bundle_type, aoi_coordinates):\n",
    "    payload = {\n",
    "        \"name\": item_ids[0],\n",
    "        \"source_type\": \"scenes\",\n",
    "        \"products\": [\n",
    "            {\n",
    "                \"item_ids\": item_ids,\n",
    "                \"item_type\": item_type,\n",
    "                \"product_bundle\": bundle_type\n",
    "            }\n",
    "        ],\n",
    "        \"tools\": [\n",
    "            {\n",
    "                \"clip\": {\n",
    "                    \"aoi\": {\n",
    "                        \"type\": \"Polygon\",\n",
    "                        \"coordinates\": aoi_coordinates\n",
    "                    }\n",
    "                }\n",
    "            }\n",
    "        ]\n",
    "    }\n",
    "    return payload\n",
    "\n",
    "def order_now(payload,apiKey):\n",
    "    orders_url = 'https://api.planet.com/compute/ops/orders/v2'\n",
    "    response = requests.post(orders_url, data=json.dumps(payload), auth=HTTPBasicAuth(apiKey, ''), headers=headers)\n",
    "    print(response)\n",
    "\n",
    "    if response.status_code==202:\n",
    "        order_id =response.json()['id']\n",
    "        url = f\"https://api.planet.com/compute/ops/orders/v2/{order_id}\"\n",
    "        feature_check = requests.get(url, auth=HTTPBasicAuth(apiKey, ''))\n",
    "        if feature_check.status_code==200:\n",
    "            print(f\"Submitted a total of {len(feature_check.json()['products'][0]['item_ids'])} image ids: accepted a total of {len(feature_check.json()['products'][0]['item_ids'])} ids\")\n",
    "            print(f\"Order URL: https://api.planet.com/compute/ops/orders/v2/{order_id}\")\n",
    "            return f\"https://api.planet.com/compute/ops/orders/v2/{order_id}\"\n",
    "    else:\n",
    "        print(f'Failed with Exception code : {response.status_code}')\n",
    "        \n",
    "def download_results(order_url,folder, overwrite=False):\n",
    "    r = requests.get(order_url, auth=HTTPBasicAuth(apiKey, ''))\n",
    "    try:\n",
    "        if r.status_code ==200:\n",
    "            response = r.json()\n",
    "            results = response['_links']['results']\n",
    "            results_urls = [r['location'] for r in results]\n",
    "            results_names = [r['name'] for r in results]\n",
    "            print('{} items to download'.format(len(results_urls)))\n",
    "\n",
    "            for url, name in zip(results_urls, results_names):\n",
    "                path = pathlib.Path(os.path.join(folder,name))\n",
    "\n",
    "                if overwrite or not path.exists():\n",
    "                    print('downloading {} to {}'.format(name, path))\n",
    "                    r = requests.get(url, allow_redirects=True)\n",
    "                    path.parent.mkdir(parents=True, exist_ok=True)\n",
    "                    open(path, 'wb').write(r.content)\n",
    "                else:\n",
    "                    print('{} already exists, skipping {}'.format(path, name))\n",
    "        else:\n",
    "            print(f'Failed with response {r.status_code}')\n",
    "    except:\n",
    "        print('data not ready yet')\n",
    "    r.close()\n",
    "    # except Exception as e:\n",
    "    #     print(e)\n",
    "    #     print(order_url)\n",
    "    #     raise Exception\n",
    "    # r.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a3abdae-bd97-4d78-92d4-630f38ec7f3b",
   "metadata": {},
   "source": [
    "## DATA DOWNLOAD SCRIPTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c4eaf7c-324b-4513-a881-ca1d2d822564",
   "metadata": {},
   "outputs": [],
   "source": [
    "# sanity check: plot to ensure that the domain shape makes sense\n",
    "domain_geometry = shape(domain['config'][0]['config'])\n",
    "domain_geometry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa26b9ec-0f37-42c6-a174-9e019c4ba92a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Search using the API, and request objects\n",
    "search_endpoint_request = {\n",
    "  \"item_types\": [item_type],\n",
    "  \"filter\": domain\n",
    "}\n",
    "result = \\\n",
    "  requests.post(\n",
    "    'https://api.planet.com/data/v1/quick-search',\n",
    "    auth=HTTPBasicAuth(apiKey, ''),\n",
    "    json=search_endpoint_request)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f8a0475-ddf8-4ec9-b7d5-dc74d01c855d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# view available data and prepare the list of planet IDs to download\n",
    "geojson_data = result.json()\n",
    "gdf = gpd.GeoDataFrame.from_features(geojson_data['features'])\n",
    "\n",
    "# Add a new column to the geodataframe that includes the intersection area\n",
    "gdf['intersection_area'] = gdf['geometry'].intersection(domain_geometry).area\n",
    "\n",
    "# Calculate the percentage overlap\n",
    "gdf['overlap_percentage'] = (gdf['intersection_area'] / domain_geometry.area) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dcfb9b60-e23e-42c3-ac41-4cda633bbeae",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# prep the ID and geometry lists\n",
    "# filter to only those that overlap >= 99% of the selected domain\n",
    "id_list = [feature['id'] for idx, feature in enumerate(geojson_data['features']) if gdf['overlap_percentage'].iloc[idx] >= 99]\n",
    "geom_list = [feature['geometry'] for idx, feature in enumerate(geojson_data['features']) if gdf['overlap_percentage'].iloc[idx] >= 99]\n",
    "print('length of the data record: ', len(id_list))\n",
    "# print(sorted(id_list))\n",
    "\n",
    "# save the dates for the specific identifiers\n",
    "idDt = []\n",
    "for idd in id_list:\n",
    "    idDt.append(idd.split('_')[0])\n",
    "idDt = np.array(idDt)\n",
    "# \n",
    "print('number of dates: ',len(idDt))\n",
    "print('number of unique dates: ',len(np.unique(idDt)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca61db89-f6ad-4cb9-a1ce-ee034cd26afa",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# prepare and submit the orders\n",
    "order_urls = pd.DataFrame(columns = [\"index\",\"ID_geom\", \"order_url\"])\n",
    "\n",
    "# loop through each order payload, and submit\n",
    "for idx,IDD in enumerate(id_list):\n",
    "    print(idx,IDD)\n",
    "    \n",
    "    payload = build_payload([IDD],item_type,bundle_type,domain['config'][0]['config']['coordinates'])\n",
    "    order_url = order_now(payload,apiKey)\n",
    "    \n",
    "    order_urls.loc[idx, \"index\"] = idx        \n",
    "    order_urls.loc[idx, \"ID_geom\"] = IDD\n",
    "    order_urls.loc[idx, \"order_url\"] = order_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ee29d48-899c-40d7-8d79-8f38edf90026",
   "metadata": {},
   "outputs": [],
   "source": [
    "# check out the data, save to a csv if you want to come back later\n",
    "print(order_urls)\n",
    "order_urls.to_csv('urlSaver.csv', index = None)# save all URLs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9c224a2-52e0-488e-aa80-a033bd45c9d7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# download the orders once ready\n",
    "# outputs of \"data not ready yet\" mean that the orders need more time to process before downloading\n",
    "for url in order_urls.itertuples():\n",
    "    print(url.index,url.order_url)\n",
    "    print(\"start downloading data to\".format(), out_direc + url.ID_geom)\n",
    "    if url.order_url != None:\n",
    "        try:\n",
    "            nantest = ~np.isnan(url.order_url)\n",
    "        except:\n",
    "            download_results(url.order_url,folder = out_direc + url.ID_geom)\n",
    "    # break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45516cae-d60c-468b-886f-425fc0930ab1",
   "metadata": {},
   "outputs": [],
   "source": [
    "### ONLY RUN THIS CELL IF YOU WANT TO CHECK ON THE ORDER STATUS\n",
    "\n",
    "# see the status of the requested tiles. Are they \"active\"?\n",
    "for IDD in id_list:\n",
    "    print(IDD)\n",
    "    command = 'curl -L -H \"Authorization: api-key '+apiKey+'\"'\n",
    "    sublink = \" 'https://api.planet.com/data/v1/item-types/\"+item_type+\"/items/\"+IDD+\"/assets/' \"\n",
    "    command = command+sublink+'| jq .'+asset_type+'.status'\n",
    "    status = subprocess.run(command, shell=True)\n",
    "    print(command)\n",
    "    # break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pangeo",
   "language": "python",
   "name": "pangeo"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}