init

.gitignore

@@ -0,0 +1,10 @@
+*.pyc
+.vscode
+.idea
+.DS_Store
+.nfs*
+__pycache__
+*.pth
+output
+build
+shell

README.md

@@ -0,0 +1,105 @@
+<a id="readme-top"></a>
+
+<!-- PROJECT LOGO -->
+<div align="center">
+  <img src="assets/hugsim.png" alt="Logo" width="300">
+
+  <p>
+    <a href="https://xdimlab.github.io/HUGSIM/">
+      <img src="https://img.shields.io/badge/Project-Page-green?style=for-the-badge" alt="Project Page" height="20">
+    </a>
+    <a href="https://arxiv.org/abs/2412.01718">
+      <img src="https://img.shields.io/badge/arXiv-Paper-red?style=for-the-badge" alt="arXiv Paper" height="20">
+    </a>
+  </p>
+
+
+  <br>
+
+  <p align="left">
+    This is the official project repository of the paper <b>HUGSIM: A Real-Time, Photo-Realistic and Closed-Loop Simulator for Autonomous Driving</b>
+  </p>
+</div>
+
+---
+
+# Installation
+
+Requirements are not in principle hard requirements, but there might be some differences (not tested):
+ - Linux
+ - Python 3.11
+ - CUDA 11.8
+ - colmap 3.10-dev
+ - Pytorch 2.5.1 / Pytorch 2.2.0
+
+Create conda environment needed to run HUGSIM with:
+``` bash
+conda create --name hugsim python=3.11
+```
+
+Please install these packages following official instructions
+ - [Pytorch](https://pytorch.org/)
+ - [simple_waymo_open_dataset_reader](https://github.com/gdlg/simple-waymo-open-dataset-reader)
+ - [tinycudann](https://github.com/NVlabs/tiny-cuda-nn)
+ - [unidepth](https://github.com/lpiccinelli-eth/UniDepth)
+ - [flow_vis_torch](https://github.com/ChristophReich1996/Optical-Flow-Visualization-PyTorch)
+ - [pytorch3d](https://github.com/facebookresearch/pytorch3d/blob/main/INSTALL.md)
+ - [kitti360Scripts](https://github.com/autonomousvision/kitti360Scripts)
+ - [nuscenes-devkit](https://github.com/nutonomy/nuscenes-devkit) (have to install from the source code for python 3.11)
+ - [HUGSIM_splat](https://github.com/hyzhou404/HUGSIM_splat)
+
+Please install packages required by [InverseForm](https://github.com/Qualcomm-AI-research/InverseForm/blob/main/docker/Dockerfile)
+
+Install remaining dependencies by:
+``` bash
+pip install -r requirements.txt
+```
+
+# Data Preparation
+
+Please refer to [Data Preparation Document](data/README.md)
+
+We will provide sample sequence data and reconstructed results. The download link will be provided later.
+
+# Reconstruction
+
+``` bash
+seq=${seq_name}
+input_path=${datadir}/${seq}
+output_path=${modeldir}/${seq}
+mkdir -p ${output_path}
+CUDA_VISIBLE_DEVICES=4 \
+python -u train_ground.py --data_cfg ./configs/${dataset_name: [kitti360, waymo, nusc, pandaset]}.yaml \
+        --source_path ${input_path} --model_path ${output_path}
+CUDA_VISIBLE_DEVICES=4 \
+python -u train.py --data_cfg ./configs/${dataset_name}.yaml \
+        --source_path ${input_path} --model_path ${output_path}
+```
+
+# Simulation
+
+**Before simulation, [UniAD_SIM](https://github.com/hyzhou404/UniAD_SIM), [VAD_SIM](https://github.com/hyzhou404/VAD_SIM) and [NAVSIM](https://github.com/hyzhou404/NAVSIM) client should be installed.**
+
+``` bash
+CUDA_VISIBLE_DEVICES=${sim_cuda} \
+python closed_loop.py --scenario_path ${scenario_cfg_path} \
+            --base_path ./configs/sim/${dataset_name}_base.yaml \
+            --camera_path ./configs/sim/${dataset_name}_camera.yaml \
+            --kinematic_path ./configs/sim/kinematic.yaml \
+            --ad ${method_name: [uniad, vad, ltf]} \
+            --ad_cuda ${ad_cuda}
+```
+
+
+# Citation
+
+If you find our paper and codes useful, please kindly cite us via:
+
+```bibtex
+@article{zhou2024hugsim,
+  title={HUGSIM: A Real-Time, Photo-Realistic and Closed-Loop Simulator for Autonomous Driving},
+  author={Zhou, Hongyu and Lin, Longzhong and Wang, Jiabao and Lu, Yichong and Bai, Dongfeng and Liu, Bingbing and Wang, Yue and Geiger, Andreas and Liao, Yiyi},
+  journal={arXiv preprint arXiv:2412.01718},
+  year={2024}
+}
+```

closed_loop.py

@@ -0,0 +1,140 @@
+import sys
+import os
+sys.path.append(os.getcwd())
+
+import gymnasium
+import hugsim_env
+from argparse import ArgumentParser
+from sim.utils.sim_utils import traj2control, traj_transform_to_global
+import pickle
+import json
+import pickle
+from sim.utils.launch_ad import launch, check_alive
+from omegaconf import OmegaConf
+import open3d as o3d
+from sim.utils.score_calculator import hugsim_evaluate
+import numpy as np
+
+
+def create_gym_env(cfg, output):
+
+    env = gymnasium.make('hugsim_env/HUGSim-v0', cfg=cfg, output=output)
+
+    observations_save, infos_save = [], []
+    obs, info = env.reset()
+    done = False
+    cnt = 0
+    save_data = {'type': 'closeloop', 'frames': []}
+
+    obs_pipe = os.path.join(output, 'obs_pipe')
+    plan_pipe = os.path.join(output, 'plan_pipe')
+    if not os.path.exists(obs_pipe):
+        os.mkfifo(obs_pipe)
+    if not os.path.exists(plan_pipe):
+        os.mkfifo(plan_pipe)
+    print('Ready for simulation')
+
+    obs, info = None, None
+    while not done:
+
+        if obs is None or info is None:
+            obs, info = env.reset()
+
+        print('ego pose', info['ego_pos'])
+
+        with open(obs_pipe, "wb") as pipe:
+            pipe.write(pickle.dumps((obs, info)))
+        with open(plan_pipe, "rb") as pipe:
+            plan_traj = pickle.loads(pipe.read())
+
+        if plan_traj is not None:
+            acc, steer_rate = traj2control(plan_traj, info)
+
+            action = {'acc': acc, 'steer_rate': steer_rate}
+            obs, reward, terminated, truncated, info = env.step(action)
+            cnt += 1
+            done = terminated or truncated or cnt > 400
+
+        else:  # AD Side Crushed
+            observations_save.append(obs)
+            infos_save.append(info)
+            done = True
+
+        imu_plan_traj = plan_traj[:, [1, 0]]
+        imu_plan_traj[:, 1] *= -1
+        global_traj = traj_transform_to_global(imu_plan_traj, info['ego_box'])
+        save_data['frames'].append({
+            'time_stamp': info['timestamp'],
+            'is_key_frame': True,
+            'ego_box': info['ego_box'],
+            'obj_boxes': info['obj_boxes'],
+            'obj_names': ['car' for _ in info['obj_boxes']],
+            'planned_traj': {
+                'traj': global_traj,
+                'timestep': 0.5
+            },
+            'collision': info['collision'],
+            'rc': info['rc']
+        })
+
+    with open(obs_pipe, "wb") as pipe:
+        pipe.write(pickle.dumps('Done'))
+
+    with open(os.path.join(output, 'data.pkl'), 'wb') as wf:
+        pickle.dump([save_data], wf)
+
+    ground_xyz = np.asarray(o3d.io.read_point_cloud(os.path.join(output, 'ground.ply')).points)
+    scene_xyz = np.asarray(o3d.io.read_point_cloud(os.path.join(output, 'scene.ply')).points)
+    results = hugsim_evaluate([save_data], ground_xyz, scene_xyz)
+    with open(os.path.join(output, 'eval.json'), 'w') as f:
+        json.dump(results, f)
+
+
+if __name__ == "__main__":
+    # Set up command line argument parser
+    parser = ArgumentParser(description="Testing script parameters")
+    parser.add_argument("--scenario_path", type=str, required=True)
+    parser.add_argument("--base_path", type=str, required=True)
+    parser.add_argument("--camera_path", type=str, required=True)
+    parser.add_argument("--kinematic_path", type=str, required=True)
+    parser.add_argument('--ad', default="uniad")
+    parser.add_argument('--ad_cuda', default="1")
+    args = parser.parse_args()
+
+    scenario_config = OmegaConf.load(args.scenario_path)
+    base_config = OmegaConf.load(args.base_path)
+    camera_config = OmegaConf.load(args.camera_path)
+    kinematic_config = OmegaConf.load(args.kinematic_path)
+    cfg = OmegaConf.merge(
+        {"scenario": scenario_config},
+        {"base": base_config},
+        {"camera": camera_config},
+        {"kinematic": kinematic_config}
+    )
+    cfg.base.output_dir = cfg.base.output_dir + args.ad
+
+    model_path = os.path.join(cfg.base.model_base, cfg.scenario.scene_name)
+    model_config = OmegaConf.load(os.path.join(model_path, 'cfg.yaml'))
+    cfg.update(model_config)
+
+    output = os.path.join(cfg.base.output_dir, cfg.scenario.scene_name+"_"+cfg.scenario.mode)
+    os.makedirs(output, exist_ok=True)
+
+    if args.ad == 'uniad':
+        ad_path = cfg.base.uniad_path
+    elif args.ad == 'vad':
+        ad_path = cfg.base.vad_path
+    elif args.ad == 'ltf':
+        ad_path = cfg.base.ltf_path
+    else:
+        raise NotImplementedError
+    process = launch(ad_path, args.ad_cuda, output)
+    try:
+        create_gym_env(cfg, output)
+        check_alive(process)
+    except Exception as e:
+        print(e)
+        process.kill()
+
+    # # For debug
+    # create_gym_env(cfg, output)

configs/benchmark/nuscenes/scene-0383-easy-00.yaml

@@ -0,0 +1,14 @@
+mode: easy_00
+plan_list: []
+load_HD_map: false
+start_euler:
+- 0.0
+- 0.0
+- 0.0
+start_ab:
+- 0.0
+- 0.0
+start_velo: 1
+start_steer: 0
+scene_name: scene-0383
+iteration: 30000

configs/benchmark/nuscenes/scene-0383-extreme-00.yaml

@@ -0,0 +1,32 @@
+mode: extreme_00
+plan_list:
+- - 0.0
+  - -20.0
+  - -0.3
+  - 0
+  - 3.0
+  - "2024_06_04_15_45_41"
+  - AttackPlanner
+  - pred_steps: 20
+    ATTACK_FREQ: 10
+    best_k: 10
+- - 2.0
+  - 20.0
+  - -0.3
+  - 0
+  - 0
+  - "2024_07_02_14_14_52"
+  - ConstantPlanner
+  - {}
+load_HD_map: false
+start_euler:
+- 0.0
+- 0.0
+- 0.0
+start_ab:
+- 0.0
+- 0.0
+start_velo: 1
+start_steer: 0
+scene_name: scene-0383
+iteration: 30000

configs/benchmark/nuscenes/scene-0383-hard-00.yaml

@@ -0,0 +1,30 @@
+mode: hard_00
+plan_list:
+- - 2.2
+  - 13.0
+  - -0.3
+  - 0.02
+  - 2.0
+  - "2024_07_05_10_58_02"
+  - ConstantPlanner
+  - {}
+- - 0.0
+  - 10.0
+  - -0.3
+  - 0.02
+  - 2.0
+  - "2024_07_05_16_10_02"
+  - ConstantPlanner
+  - {}
+load_HD_map: false
+start_euler:
+- 0.0
+- 0.0
+- 0.0
+start_ab:
+- 0.0
+- 0.0
+start_velo: 1
+start_steer: 0
+scene_name: scene-0383
+iteration: 30000

configs/benchmark/nuscenes/scene-0383-medium-00.yaml

@@ -0,0 +1,22 @@
+mode: medium_00
+plan_list:
+- - 2.0
+  - 15.0
+  - -0.3
+  - 0
+  - 0
+  - "2024_07_09_14_25_36"
+  - ConstantPlanner
+  - {}
+load_HD_map: false
+start_euler:
+- 0.0
+- 0.0
+- 0.0
+start_ab:
+- 0.0
+- 0.0
+start_velo: 1
+start_steer: 0
+scene_name: scene-0383
+iteration: 30000

configs/benchmark/nuscenes/scene-0655-easy-00.yaml

@@ -0,0 +1,14 @@
+mode: easy_00
+plan_list: []
+load_HD_map: false
+start_euler:
+- 0.0
+- 0.0
+- 0.0
+start_ab:
+- 0.0
+- 0.0
+start_velo: 1
+start_steer: 0
+scene_name: scene-0655
+iteration: 30000