notebooks/util.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If Dockerfiles have not been modified, connect to the Jupyter server with ```http://localhost:8000/tree?token=util```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following block takes the base Place Pulse Singapore data file (```place-pulse-singapore.csv``` by default) and generates a ```.csv``` of unique location IDs and their corresponding latitudes and longitudes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_dir = \"data\"\n",
    "input = \"place-pulse-singapore.csv\"\n",
    "output = \"place-pulse-singapore-locations.csv\"\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import os\n",
    "\n",
    "df = pd.read_csv(os.path.join(target_dir, input))\n",
    "df.columns = [\"id\", \"location_id\", \"lat\", \"lon\", \"num_votes\", \"perception\", \"trueskill_score\", \"trueskill_stds\"]\n",
    "df = df.drop_duplicates(subset=[\"location_id\"])[[\"location_id\", \"lat\", \"lon\"]]\n",
    "df.columns = [\"id\", \"lat\", \"lon\"]\n",
    "with open(os.path.join(target_dir, output), 'w') as fp:\n",
    "    df.to_csv(fp, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following block takes the base Place Pulse Singapore data file (```place-pulse-singapore.csv``` by default), and for each unique location, attempts to find its ```trueskill_score``` for each ```perception```.  \n",
    "One ```.json``` is created for each perception."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_dir = \"data\"\n",
    "place_pulse_singapore_file = \"place-pulse-singapore.csv\"\n",
    "output_dir = \"place-pulse-singapore-labels\"\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "with open(os.path.join(target_dir, place_pulse_singapore_file), 'r') as fp:\n",
    "    place_pulse_singapore_df = pd.read_csv(fp)\n",
    "\n",
    "Path(os.path.join(target_dir, output_dir)).mkdir(parents=True, exist_ok=True)\n",
    "for perception, indices in place_pulse_singapore_df.groupby(by=\"perception\").groups.items():\n",
    "    perception_df = place_pulse_singapore_df.iloc[indices].drop_duplicates(subset=\"location_id\")[[\"location_id\", \"trueskill_score\"]]\n",
    "    with open(os.path.join(target_dir, output_dir, '-'.join(perception.split(' ')) + \".csv\"), 'w') as fp:\n",
    "        perception_df.to_csv(fp, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following block takes a m×1 and n×1 matrices as ```.json```s and combines them into a (n+m)×1 matrix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_dir = \"data\"\n",
    "input1_dir = \"place-pulse-singapore-panos-encoded\"\n",
    "input2_dir = \"place-pulse-singapore-point-clouds-encoded\"\n",
    "output_dir = \"place-pulse-singapore-combined-encoded\"\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import json\n",
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "input1s_path = []\n",
    "for dirpath, dirnames, filenames in os.walk(os.path.join(target_dir, input1_dir)):\n",
    "    input1s_path.extend(filenames)\n",
    "    break\n",
    "input1s_id = ['.'.join(input1_path.split('.')[:-1]) for input1_path in input1s_path]\n",
    "input1s = [] \n",
    "for input1_path in input1s_path:\n",
    "    with open(os.path.join(target_dir, input1_dir, input1_path), 'r') as fp:\n",
    "        input1s.append(json.load(fp))\n",
    "input1s_df = pd.DataFrame(input1s)\n",
    "input1s_df.index = input1s_id\n",
    "\n",
    "input2s_path = []\n",
    "for dirpath, dirnames, filenames in os.walk(os.path.join(target_dir, input2_dir)):\n",
    "    input2s_path.extend(filenames)\n",
    "    break\n",
    "input2s_id = ['.'.join(input2_path.split('.')[:-1]) for input2_path in input2s_path]\n",
    "input2s = [] \n",
    "for input2_path in input2s_path:\n",
    "    with open(os.path.join(target_dir, input2_dir, input2_path), 'r') as fp:\n",
    "        input2s.append(json.load(fp))\n",
    "input2s_df = pd.DataFrame(input2s)\n",
    "input2s_df.index = input2s_id\n",
    "\n",
    "combined_df = input1s_df.join(input2s_df, how=\"inner\", lsuffix=\"_left\", rsuffix=\"_right\")\n",
    "\n",
    "Path(os.path.join(target_dir, output_dir)).mkdir(parents=True, exist_ok=True)\n",
    "for i in combined_df.index:\n",
    "    with open(os.path.join(target_dir, output_dir, i + \".json\"), 'w') as fp:\n",
    "        json.dump(combined_df.loc[i].values.flatten().tolist(), fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following block takes an image and resizes it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_dir = \"data\"\n",
    "input = \"\"\n",
    "output = \"\"\n",
    "\n",
    "from PIL import Image\n",
    "\n",
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "images_path = []\n",
    "for dirpath, dirnames, filenames in os.walk(os.path.join(target_dir, input)):\n",
    "    images_path.extend(filenames)\n",
    "    break\n",
    "images_id = {}\n",
    "for image_path in images_path:\n",
    "    images_id['.'.join(image_path.split('.')[:-1])] = os.path.join(target_dir, input, image_path)\n",
    "\n",
    "Path(os.path.join(target_dir, output)).mkdir(parents=True, exist_ok=True)\n",
    "for id, path in images_id.items():\n",
    "    image = Image.open(path)\n",
    "    if image is None:\n",
    "        continue\n",
    "    image = image.resize((512, 256))\n",
    "    image.save(os.path.join(target_dir, output, id + \".jpg\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following block takes a ```.csv``` without headers describing the cartesian (xyz) coordinates of a point cloud and returns ```.csv``` describing a point cloud with 1024 points via furthest-point sampling.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_dir = \"data\"\n",
    "input_dir = \"thomson-segmented-point-clouds-split\"\n",
    "output_dir = \"thomson-segmented-point-clouds-split-sampled\"\n",
    "\n",
    "import fpsample\n",
    "import numpy as np\n",
    "\n",
    "import csv\n",
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "point_clouds_path = []\n",
    "for dirpath, dirnames, filenames in os.walk(os.path.join(target_dir, input_dir)):\n",
    "    for filename in filenames:\n",
    "        if Path(os.path.join(dirpath, filename)).is_file():\n",
    "            point_clouds_path.append(os.path.join(dirpath, filename).split(os.path.join(target_dir, input_dir) + '/')[-1])\n",
    "\n",
    "for path in point_clouds_path:\n",
    "    filename_no_ext = path.split('.')[0]\n",
    "    if Path(os.path.join(target_dir, output_dir, filename_no_ext + \".csv\")).is_file():\n",
    "        continue\n",
    "    if not Path(os.path.dirname(os.path.join(target_dir, output_dir, filename_no_ext + \".csv\"))).is_dir():\n",
    "        Path(os.path.dirname(os.path.join(target_dir, output_dir, filename_no_ext + \".csv\"))).mkdir(parents=True, exist_ok=True)\n",
    "    point_cloud = []\n",
    "    with open(os.path.join(target_dir, input_dir, path), 'r') as fp:\n",
    "        csv_reader = csv.reader(fp, delimiter=' ')\n",
    "        for row in csv_reader:\n",
    "            point_cloud.append(row)\n",
    "    if len(point_cloud) < 1024:\n",
    "        continue\n",
    "    fps_samples_idx = fpsample.fps_sampling(np.array([(float(row[0]), float(row[1]), float(row[2])) for row in point_cloud]), 1024)\n",
    "    point_cloud_sampled = [point_cloud[i] for i in fps_samples_idx]\n",
    "    with open(os.path.join(target_dir, output_dir, filename_no_ext + \".csv\"), 'w') as fp:\n",
    "        csv_writer = csv.writer(fp)\n",
    "        csv_writer.writerows(point_cloud_sampled)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following block takes a ```.json``` of a 2D array describing a depthmap and returns ```.csv``` describing a point cloud from the depthmap.  \n",
    "It assumes the depthmap is oriented with the centre facing north.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_dir = \"data\"\n",
    "input = \"place-pulse-singapore-depths-512-1024\"\n",
    "output = \"place-pulse-singapore-depth-point-clouds\"\n",
    "\n",
    "import csv\n",
    "import json\n",
    "import math\n",
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "def depthmap_to_xyz(depthmap: list[list[float]],\n",
    "                    xrange: tuple[float, float] = (-1.0, 1.0), yrange: tuple[float, float] = (-1.0, 1.0),\n",
    "                    heading: float = 0,\n",
    "                    rmin: float = 0.0, rmax: float = math.inf) -> list[list[float]]:\n",
    "    pi = math.pi\n",
    "    sin = math.sin\n",
    "    cos = math.cos\n",
    "    output = []\n",
    "    width = len(depthmap[0])\n",
    "    height = len(depthmap)\n",
    "    x0 = xrange[0]\n",
    "    dx = xrange[1] - x0\n",
    "    y0 = yrange[0]\n",
    "    dy = yrange[1] - y0\n",
    "    h = -heading\n",
    "    for i in range(height):\n",
    "        for j in range(width):\n",
    "            r = depthmap[i][j]\n",
    "            if r < rmin or r > rmax:\n",
    "                continue\n",
    "            xnorm = ((j + 0.5) / width) * dx + x0\n",
    "            ynorm = ((i + 0.5) / height) * dy + y0\n",
    "            theta = -pi * xnorm\n",
    "            phi = -pi / 2 * ynorm\n",
    "            cartesian = [[-r * sin(h + theta) * cos(phi)],\n",
    "                         [r * cos(h + theta) * cos(phi)],\n",
    "                         [r * sin(phi)]]\n",
    "            output.append([cartesian[0][0], cartesian[1][0], cartesian[2][0]])\n",
    "    return output\n",
    "\n",
    "depths_path = []\n",
    "for dirpath, dirnames, filenames in os.walk(os.path.join(target_dir, input)):\n",
    "    depths_path.extend(filenames)\n",
    "    break\n",
    "depths_id = {}\n",
    "for depth_path in depths_path:\n",
    "    depths_id['.'.join(depth_path.split('.')[:-1])] = os.path.join(target_dir, input, depth_path)\n",
    "\n",
    "Path(os.path.join(target_dir, output)).mkdir(parents=True, exist_ok=True)\n",
    "for id, path in depths_id.items():\n",
    "    depth = None\n",
    "    with open(path, 'r') as fp:\n",
    "        depth = json.load(fp)\n",
    "    if depth is None:\n",
    "        continue\n",
    "    point_cloud = depthmap_to_xyz(depth)\n",
    "    with open(os.path.join(target_dir, output, f\"{id}.csv\"), 'w') as fp:\n",
    "        csv_writer = csv.writer(fp)\n",
    "        csv_writer.writerows(point_cloud)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following block takes a ```.json``` of a 2D array describing a depthmap and a ```.json``` of a 2D array describing the semantic segmentation associated with that depthmap/panorama and returns ```.csv``` describing a segmented point cloud derived from the two inputs and a ```.json``` describing the class labels.  \n",
    "It assumes the inputs are oriented with the centre facing north.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_dir = \"data\"\n",
    "depthmap_dir = \"place-pulse-singapore-depths-512-1024\"\n",
    "segmentation_dir = \"place-pulse-singapore-segmented-512-1024\"\n",
    "class_id = \"classes.json\"\n",
    "output_dir = \"place-pulse-singapore-segmented-point-clouds\"\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import csv\n",
    "import json\n",
    "import math\n",
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "def depthmap_to_segmented_xyz(depthmap: list[list[float]], segmentationmap: list[list[float]],\n",
    "                    xrange: tuple[float, float] = (-1.0, 1.0), yrange: tuple[float, float] = (-1.0, 1.0),\n",
    "                    heading: float = 0,\n",
    "                    rmin: float = 0.0, rmax: float = math.inf) -> list[list[float]]:\n",
    "    pi = math.pi\n",
    "    sin = math.sin\n",
    "    cos = math.cos\n",
    "    output = []\n",
    "    width = len(depthmap[0])\n",
    "    height = len(depthmap)\n",
    "    x0 = xrange[0]\n",
    "    dx = xrange[1] - x0\n",
    "    y0 = yrange[0]\n",
    "    dy = yrange[1] - y0\n",
    "    h = -heading\n",
    "    for i in range(height):\n",
    "        for j in range(width):\n",
    "            r = depthmap[i][j]\n",
    "            if r < rmin or r > rmax:\n",
    "                continue\n",
    "            xnorm = ((j + 0.5) / width) * dx + x0\n",
    "            ynorm = ((i + 0.5) / height) * dy + y0\n",
    "            theta = -pi * xnorm\n",
    "            phi = -pi / 2 * ynorm\n",
    "            cartesian = [[-r * sin(h + theta) * cos(phi)],\n",
    "                         [r * cos(h + theta) * cos(phi)],\n",
    "                         [r * sin(phi)]]\n",
    "            output.append([cartesian[0][0], cartesian[1][0], cartesian[2][0], segmentationmap[i][j]])\n",
    "    return output\n",
    "\n",
    "segmented_depths_path = []\n",
    "for dirpath, dirnames, filenames in os.walk(os.path.join(target_dir, depthmap_dir)):\n",
    "    segmented_depths_path.extend(filenames)\n",
    "    break\n",
    "segmented_depths_id = {}\n",
    "for segmented_depth_path in segmented_depths_path:\n",
    "    segmented_depths_id['.'.join(segmented_depth_path.split('.')[:-1])] = [os.path.join(target_dir, depthmap_dir, segmented_depth_path), os.path.join(target_dir, segmentation_dir, segmented_depth_path)]\n",
    "\n",
    "Path(os.path.join(target_dir, output_dir)).mkdir(parents=True, exist_ok=True)\n",
    "for id, path in segmented_depths_id.items():\n",
    "    if Path(os.path.join(target_dir, output_dir, f\"{id}.csv\")).is_file():\n",
    "        continue\n",
    "    depth = None\n",
    "    with open(path[0], 'r') as fp:\n",
    "        depth = json.load(fp)\n",
    "    if depth is None:\n",
    "        continue\n",
    "    segmentation = None\n",
    "    with open(path[1], 'r') as fp:\n",
    "        segmentation = json.load(fp)\n",
    "    if segmentation is None or np.array(depth).shape != np.array(segmentation).shape:\n",
    "        continue\n",
    "    segmented_point_cloud = depthmap_to_segmented_xyz(depth, segmentation)\n",
    "    with open(os.path.join(target_dir, output_dir, f\"{id}.csv\"), 'w') as fp:\n",
    "        csv_writer = csv.writer(fp)\n",
    "        csv_writer.writerows(segmented_point_cloud)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following block takes a ```.jpg``` of a panorama, a ```.json``` of a 2D array describing the depthmap of the panorama, and a ```.json``` of a 2D array describing the semantic segmentation of the panorama and returns ```.csv``` describing a segmented coloured point cloud derived from the two inputs and a ```.json``` describing the class labels.  \n",
    "It assumes the inputs are oriented with the centre facing north.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_dir = \"data\"\n",
    "pano_dir = \"thomson-panos\"\n",
    "depthmap_dir = \"thomson-depths\"\n",
    "segmentation_dir = \"thomson-segmented\"\n",
    "class_id = \"classes.json\"\n",
    "output_dir = \"thomson-segmented-coloured-point-clouds\"\n",
    "\n",
    "import numpy as np\n",
    "from PIL import Image\n",
    "\n",
    "import csv\n",
    "import json\n",
    "import math\n",
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "def depthmap_to_segmented_coloured_xyz(depthmap: list[list[float]], segmentationmap: list[list[float]], pano: list[list[list[int]]],\n",
    "                    xrange: tuple[float, float] = (-1.0, 1.0), yrange: tuple[float, float] = (-1.0, 1.0),\n",
    "                    heading: float = 0,\n",
    "                    rmin: float = 0.0, rmax: float = math.inf) -> list[list[float]]:\n",
    "    pi = math.pi\n",
    "    sin = math.sin\n",
    "    cos = math.cos\n",
    "    output = []\n",
    "    width = len(depthmap[0])\n",
    "    height = len(depthmap)\n",
    "    x0 = xrange[0]\n",
    "    dx = xrange[1] - x0\n",
    "    y0 = yrange[0]\n",
    "    dy = yrange[1] - y0\n",
    "    h = -heading\n",
    "    for i in range(height):\n",
    "        for j in range(width):\n",
    "            r = depthmap[i][j]\n",
    "            if r < rmin or r > rmax:\n",
    "                continue\n",
    "            xnorm = ((j + 0.5) / width) * dx + x0\n",
    "            ynorm = ((i + 0.5) / height) * dy + y0\n",
    "            theta = -pi * xnorm\n",
    "            phi = -pi / 2 * ynorm\n",
    "            cartesian = [[-r * sin(h + theta) * cos(phi)],\n",
    "                         [r * cos(h + theta) * cos(phi)],\n",
    "                         [r * sin(phi)]]\n",
    "            output.append([cartesian[0][0], cartesian[1][0], cartesian[2][0], pano[i][j][0], pano[i][j][1], pano[i][j][2], segmentationmap[i][j]])\n",
    "    return output\n",
    "\n",
    "segmented_depths_path = []\n",
    "for dirpath, dirnames, filenames in os.walk(os.path.join(target_dir, depthmap_dir)):\n",
    "    segmented_depths_path.extend(filenames)\n",
    "    break\n",
    "segmented_depths_id = {}\n",
    "for segmented_depth_path in segmented_depths_path:\n",
    "    segmented_depths_id['.'.join(segmented_depth_path.split('.')[:-1])] = [os.path.join(target_dir, depthmap_dir, segmented_depth_path),\n",
    "                                                                           os.path.join(target_dir, segmentation_dir, segmented_depth_path),\n",
    "                                                                           os.path.join(target_dir, pano_dir, '.'.join(segmented_depth_path.split('.')[:-1]) + \".jpg\")]\n",
    "\n",
    "Path(os.path.join(target_dir, output_dir)).mkdir(parents=True, exist_ok=True)\n",
    "for id, path in segmented_depths_id.items():\n",
    "    if Path(os.path.join(target_dir, output_dir, f\"{id}.csv\")).is_file():\n",
    "        continue\n",
    "    depth = None\n",
    "    with open(path[0], 'r') as fp:\n",
    "        depth = json.load(fp)\n",
    "    if depth is None:\n",
    "        continue\n",
    "    depth_shape = np.array(depth).shape\n",
    "    segmentation = None\n",
    "    with open(path[1], 'r') as fp:\n",
    "        segmentation = json.load(fp)\n",
    "    if segmentation is None or depth_shape != np.array(segmentation).shape:\n",
    "        continue\n",
    "    image = Image.open(path[2])\n",
    "    if image is None:\n",
    "        continue\n",
    "    pano = np.array(image.resize((depth_shape[1], depth_shape[0])).getdata()).reshape((depth_shape[0], depth_shape[1], 3)).tolist()\n",
    "    segmented_coloured_point_cloud = depthmap_to_segmented_coloured_xyz(depth, segmentation, pano)\n",
    "    with open(os.path.join(target_dir, output_dir, f\"{id}.csv\"), 'w') as fp:\n",
    "        csv_writer = csv.writer(fp)\n",
    "        csv_writer.writerows(segmented_coloured_point_cloud)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following block takes a ```.csv``` describing a segmented coloured point cloud and a ```.json``` describing its class labels and converts it into a format similar to the S3DIS dataset.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_dir = \"data\"\n",
    "input_dir = \"place-pulse-singapore-segmented-coloured-point-clouds\"\n",
    "classes_id = \"classes.json\"\n",
    "output_dir = \"place-pulse-singapore-point-clouds-s3dis\"\n",
    "cityscapes_name_to_s3dis_name = {\n",
    "    \"road\": \"floor\",\n",
    "    \"sidewalk\": \"sofa\",\n",
    "    \"building\": \"wall\",\n",
    "    \"wall\":  \"wall\",\n",
    "    \"fence\": \"wall\",\n",
    "    \"pole\": \"clutter\",\n",
    "    \"traffic light\": \"clutter\",\n",
    "    \"traffic sign\": \"clutter\",\n",
    "    \"vegetation\": \"column\",\n",
    "    \"terrain\": \"table\",\n",
    "    \"sky\": \"ceiling\",\n",
    "    \"person\": \"clutter\",\n",
    "    \"rider\": \"clutter\",\n",
    "    \"car\": \"clutter\",\n",
    "    \"truck\": \"clutter\",\n",
    "    \"bus\": \"clutter\",\n",
    "    \"train\": \"clutter\",\n",
    "    \"motorcycle\": \"clutter\",\n",
    "    \"bicycle\": \"clutter\"\n",
    "}\n",
    "\n",
    "import csv\n",
    "import json\n",
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "def cityscapes_label_to_s3dis_name(label, cityscapes_name_to_s3dis_name: dict,\n",
    "                                   cityscapes_label_to_name: dict = {\n",
    "                                       0: \"road\",\n",
    "                                       1: \"sidewalk\",\n",
    "                                       2: \"building\",\n",
    "                                       3: \"wall\",\n",
    "                                       4: \"fence\",\n",
    "                                       5: \"pole\",\n",
    "                                       6: \"traffic light\",\n",
    "                                       7: \"traffic sign\",\n",
    "                                       8: \"vegetation\",\n",
    "                                       9: \"terrain\",\n",
    "                                       10: \"sky\",\n",
    "                                       11: \"person\",\n",
    "                                       12: \"rider\",\n",
    "                                       13: \"car\",\n",
    "                                       14: \"truck\",\n",
    "                                       15: \"bus\",\n",
    "                                       16: \"train\",\n",
    "                                       17: \"motorcycle\",\n",
    "                                       18: \"bicycle\"\n",
    "                                   }) -> str:\n",
    "    if type(list(cityscapes_label_to_name.keys())[0]) == str:\n",
    "        cityscapes_label = str(label)\n",
    "    elif type(list(cityscapes_label_to_name.keys())[0]) == int:\n",
    "        cityscapes_label = int(label)\n",
    "    return cityscapes_name_to_s3dis_name[cityscapes_label_to_name[cityscapes_label]]\n",
    "\n",
    "with open(os.path.join(target_dir, classes_id), 'r') as fp:\n",
    "    cityscapes_label_to_name = json.load(fp)\n",
    "\n",
    "s3dis_label_to_name = {\n",
    "    0: \"ceiling\",\n",
    "    1: \"floor\",\n",
    "    2: \"wall\",\n",
    "    3: \"beam\",\n",
    "    4: \"column\",\n",
    "    5: \"window\",\n",
    "    6: \"door\",\n",
    "    7: \"chair\",\n",
    "    8: \"table\",\n",
    "    9: \"bookscapes\",\n",
    "    10: \"sofa\",\n",
    "    11: \"board\",\n",
    "    12: \"clutter\"\n",
    "}\n",
    "\n",
    "segmented_coloured_point_clouds_path = []\n",
    "for dirpath, dirnames, filenames in os.walk(os.path.join(target_dir, input_dir)):\n",
    "    segmented_coloured_point_clouds_path.extend(filenames)\n",
    "    break\n",
    "segmented_coloured_point_clouds_id = {}\n",
    "for segmented_coloured_point_cloud_path in segmented_coloured_point_clouds_path:\n",
    "    segmented_coloured_point_clouds_id['.'.join(segmented_coloured_point_cloud_path.split('.')[:-1])] = os.path.join(target_dir, input_dir, segmented_coloured_point_cloud_path)\n",
    "\n",
    "Path(os.path.join(target_dir, output_dir)).mkdir(parents=True, exist_ok=True)\n",
    "for id, path in segmented_coloured_point_clouds_id.items():\n",
    "    if Path(os.path.join(target_dir, output_dir, id, id, \"Annotations\")).is_dir():\n",
    "        continue\n",
    "    Path(os.path.join(target_dir, output_dir, id, id, \"Annotations\")).mkdir(parents=True, exist_ok=True)\n",
    "    rows = []\n",
    "    with open(path, 'r') as fp:\n",
    "        reader = csv.reader(fp)\n",
    "        for row in reader:\n",
    "            rows.append(row)\n",
    "\n",
    "    with open(os.path.join(target_dir, output_dir, id, id, f\"{id}.txt\"), 'w') as fp:\n",
    "        point_cloud_writer = csv.writer(fp, delimiter=' ')\n",
    "        point_cloud_writer.writerows([row[:-1] for row in rows])\n",
    "\n",
    "    s3dis_names = set()\n",
    "    class_rows = {}\n",
    "    for row in rows:\n",
    "        s3dis_name = cityscapes_label_to_s3dis_name(row[6], cityscapes_name_to_s3dis_name, cityscapes_label_to_name=cityscapes_label_to_name) + \"_1\"\n",
    "        if not s3dis_name in s3dis_names:\n",
    "            s3dis_names.add(s3dis_name)\n",
    "            class_rows[s3dis_name] = [row[:-1]]\n",
    "        else:\n",
    "            class_rows[s3dis_name].append(row[:-1])\n",
    "    \n",
    "    for name, rows in class_rows.items():\n",
    "        with open(os.path.join(target_dir, output_dir, id, id, \"Annotations\", f\"{name}.txt\"), 'w') as fp:\n",
    "            segment_writer = csv.writer(fp, delimiter=' ')\n",
    "            segment_writer.writerows(rows)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following block takes a ```.csv``` describing a segmented (optionally coloured) point cloud and a ```.json``` describing its class labels and splits it separate point clouds for each label.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_dir = \"data\"\n",
    "input_dir = \"thomson-segmented-coloured-point-clouds\"\n",
    "classes_id = \"classes.json\"\n",
    "output_dir = \"thomson-segmented-point-clouds-split\"\n",
    "cityscapes_label_to_name = {\n",
    "    0: \"road\",\n",
    "    1: \"sidewalk\",\n",
    "    2: \"building\",\n",
    "    3: \"wall\",\n",
    "    4: \"fence\",\n",
    "    5: \"pole\",\n",
    "    6: \"traffic light\",\n",
    "    7: \"traffic sign\",\n",
    "    8: \"vegetation\",\n",
    "    9: \"terrain\",\n",
    "    10: \"sky\",\n",
    "    11: \"person\",\n",
    "    12: \"rider\",\n",
    "    13: \"car\",\n",
    "    14: \"truck\",\n",
    "    15: \"bus\",\n",
    "    16: \"train\",\n",
    "    17: \"motorcycle\",\n",
    "    18: \"bicycle\"\n",
    "}\n",
    "cityscapes_name_to_split = {\n",
    "    \"road\": \"road\",\n",
    "    \"sidewalk\": \"sidewalk\",\n",
    "    \"building\": \"building\",\n",
    "    \"wall\": \"building\",\n",
    "    \"fence\": \"building\",\n",
    "    \"vegetation\": \"vegetation\",\n",
    "    \"terrain\": \"terrain\"\n",
    "}\n",
    "\n",
    "import csv\n",
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "segmented_coloured_point_clouds_path = []\n",
    "for dirpath, dirnames, filenames in os.walk(os.path.join(target_dir, input_dir)):\n",
    "    segmented_coloured_point_clouds_path.extend(filenames)\n",
    "    break\n",
    "segmented_coloured_point_clouds_id = {}\n",
    "for segmented_coloured_point_cloud_path in segmented_coloured_point_clouds_path:\n",
    "    segmented_coloured_point_clouds_id['.'.join(segmented_coloured_point_cloud_path.split('.')[:-1])] = os.path.join(target_dir, input_dir, segmented_coloured_point_cloud_path)\n",
    "\n",
    "Path(os.path.join(target_dir, output_dir)).mkdir(parents=True, exist_ok=True)\n",
    "for id, path in segmented_coloured_point_clouds_id.items():\n",
    "    if Path(os.path.join(target_dir, output_dir, id)).is_dir():\n",
    "        continue\n",
    "    rows = []\n",
    "    with open(path, 'r') as fp:\n",
    "        reader = csv.reader(fp)\n",
    "        for row in reader:\n",
    "            rows.append(row)\n",
    "    names = set()\n",
    "    name_rows = {}\n",
    "    for row in rows:\n",
    "        cityscapes_name = cityscapes_label_to_name[int(row[-1])]\n",
    "        if not cityscapes_name in cityscapes_name_to_split:\n",
    "            continue\n",
    "        split_name = cityscapes_name_to_split[cityscapes_name]\n",
    "        if not split_name in names:\n",
    "            names.add(split_name)\n",
    "            name_rows[split_name] = [row[:-1]]\n",
    "        else:\n",
    "            name_rows[split_name].append(row[:-1])\n",
    "    Path(os.path.join(target_dir, output_dir, id)).mkdir(parents=True, exist_ok=True)\n",
    "    for name, rows in name_rows.items():\n",
    "        with open(os.path.join(target_dir, output_dir, id, f\"{name}.csv\"), 'w') as fp:\n",
    "            writer = csv.writer(fp, delimiter=' ')\n",
    "            writer.writerows(rows)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following block takes five *n*D vectors representing the point cloud encodings for road, sidewalk, building, vegetation, and terrain as  ```.json```s and combines them into a 5*n*D vector. Where any of the encodings are missing, a *n*D zero vector is used."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_dir = \"data\"\n",
    "input_dir = \"thomson-segmented-point-clouds-split-encoded\"\n",
    "output_dir = \"thomson-segmented-point-clouds-combined-encoded\"\n",
    "n = 512\n",
    "\n",
    "import json\n",
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "encodings_path = []\n",
    "for dirpath, dirnames, filenames in os.walk(os.path.join(target_dir, input_dir)):\n",
    "    for filename in filenames:\n",
    "        if Path(os.path.join(dirpath, filename)).is_file():\n",
    "            encodings_path.append(os.path.join(dirpath, filename).split(os.path.join(target_dir, input_dir) + '/')[-1])\n",
    "\n",
    "id_encodings = {}\n",
    "for path in encodings_path:\n",
    "    paths = path.split('/')\n",
    "    id = paths[-2]\n",
    "    encoding_type = '.'.join(paths[-1].split('.')[:-1])\n",
    "    if not id in id_encodings:\n",
    "        id_encodings[id] = {}\n",
    "    id_encodings[id][encoding_type] = path\n",
    "\n",
    "Path(os.path.join(target_dir, output_dir)).mkdir(parents=True, exist_ok=True)\n",
    "for id, encodings in id_encodings.items():\n",
    "    if \"road\" in encodings:\n",
    "        with open(os.path.join(target_dir, input_dir, encodings[\"road\"]), 'r') as fp:\n",
    "            road_encoding = json.load(fp)\n",
    "    else:\n",
    "        road_encoding = [0] * n\n",
    "    if \"sidewalk\" in encodings:\n",
    "        with open(os.path.join(target_dir, input_dir, encodings[\"sidewalk\"]), 'r') as fp:\n",
    "            sidewalk_encoding = json.load(fp)\n",
    "    else:\n",
    "        sidewalk_encoding = [0] * n\n",
    "    if \"building\" in encodings:\n",
    "        with open(os.path.join(target_dir, input_dir, encodings[\"building\"]), 'r') as fp:\n",
    "            building_encoding = json.load(fp)\n",
    "    else:\n",
    "        building_encoding = [0] * n\n",
    "    if \"vegetation\" in encodings:\n",
    "        with open(os.path.join(target_dir, input_dir, encodings[\"vegetation\"]), 'r') as fp:\n",
    "            vegetation_encoding = json.load(fp)\n",
    "    else:\n",
    "        vegetation_encoding = [0] * n\n",
    "    if \"terrain\" in encodings:\n",
    "        with open(os.path.join(target_dir, input_dir, encodings[\"terrain\"]), 'r') as fp:\n",
    "            terrain_encoding = json.load(fp)\n",
    "    else:\n",
    "        terrain_encoding = [0] * n\n",
    "    encoding = road_encoding + sidewalk_encoding + building_encoding + vegetation_encoding + terrain_encoding\n",
    "    with open(os.path.join(target_dir, output_dir, f\"{id}.json\"), 'w') as fp:\n",
    "        json.dump(encoding, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "with open(\"data/thomson-id.csv\", 'r') as fp:\n",
    "    id = pd.read_csv(fp)\n",
    "with open(\"data/thomson-segmented-point-clouds-combined-encoded-clustering/clustering-segmented-point-clouds-combined-encoded.csv\", 'r') as fp:\n",
    "    morphology = pd.read_csv(fp)\n",
    "with open(\"data/thomson-perceptions-clustering/clustering-perceptions.csv\", 'r') as fp:\n",
    "    perception = pd.read_csv(fp)\n",
    "\n",
    "clusters = morphology.merge(perception, on=\"id\", how=\"inner\", suffixes=(\"_morphology\", \"_perception\")).merge(id, on=\"id\", how=\"inner\")\n",
    "with open(\"data/thomson-clusters.csv\", 'w') as fp:\n",
    "    clusters.to_csv(fp, index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}