Utils/build_dataset_path_file.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "03da91bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from pathlib import Path\n",
    "import glob"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53a0ed89",
   "metadata": {},
   "source": [
    "#### - Define source data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "251fdaa7",
   "metadata": {},
   "outputs": [],
   "source": [
    "coco_dataset_root_folder = './COCO2017'\n",
    "crowd_human_dataset_root_folder = './CrowdHuman'\n",
    "source_data_path = [os.path.join(coco_dataset_root_folder, 'train2017.txt'), \\\n",
    "                    os.path.join(crowd_human_dataset_root_folder, 'CrowdHuman_train01'), \\\n",
    "                    os.path.join(crowd_human_dataset_root_folder, 'CrowdHuman_train02'), \\\n",
    "                    os.path.join(crowd_human_dataset_root_folder, 'CrowdHuman_train03')]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a47e0daf",
   "metadata": {},
   "source": [
    "#### - Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4cef8682",
   "metadata": {},
   "outputs": [],
   "source": [
    "total_data_path_list = []\n",
    "save_path = './total_data_path_list.txt'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "177e60c8",
   "metadata": {},
   "source": [
    "#### - Build total dataset path list extracted from various source data folder/text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "29f879fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---- path 1: COCO2017/train2017.txt\n",
      "-- Text File\n",
      "-- Length of file_path_list: 118287\n",
      "---- path 2: CrowdHuman/CrowdHuman_train01\n",
      "-- Directory\n",
      "-- Length of file_path_list: 5000\n",
      "---- path 3: CrowdHuman/CrowdHuman_train02\n",
      "-- Directory\n",
      "-- Length of file_path_list: 5000\n",
      "---- path 4: CrowdHuman/CrowdHuman_train03\n",
      "-- Directory\n",
      "-- Length of file_path_list: 5000\n"
     ]
    }
   ],
   "source": [
    "# Ref: dataset.py in https://github.com/WongKinYiu/yolov7\n",
    "for i, p in enumerate(source_data_path) if isinstance(source_data_path, list) else [source_data_path]:\n",
    "    p = Path(p)\n",
    "    print(f'---- path {i+1}: {p}')\n",
    "    if p.is_dir(): # dir\n",
    "        kind = 'Directory'\n",
    "        file_path_list = glob.glob(str(p / '**' / '*.*'), recursive=True)\n",
    "    elif p.is_file():  # file\n",
    "        kind = 'Text File'\n",
    "        with open(p, 'r') as t:\n",
    "            t = t.read().strip().splitlines()\n",
    "            parent = str(p.parent) + os.sep\n",
    "            file_path_list = [x.replace('./', parent) if x.startswith('./') else x for x in t]\n",
    "    else:\n",
    "        raise Exception(f'{prefix}{p} does not exist')\n",
    "   \n",
    "    print(f'-- {kind}')\n",
    "    print(f'-- Length of file_path_list: {len(file_path_list)}')\n",
    "    \n",
    "    total_data_path_list += file_path_list"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1fe20b8",
   "metadata": {},
   "source": [
    "#### - Write results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5b2052af",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-- Length of final_file_path_list: 133287\n"
     ]
    }
   ],
   "source": [
    "print(f'-- Length of final_file_path_list: {len(total_data_path_list)}')\n",
    "with open(save_path, mode='w', encoding='utf-8') as file:\n",
    "    for total_data_path in total_data_path_list:\n",
    "        file.write(total_data_path + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a077191b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}