bug on kitti 0002
Closed this issue · 5 comments
Hi, thanks for sharing your great work.
There is only one file under the directory of car-nerf-state-dict.
mars/latents/KITTI-MOT/car-nerf-state-dict/epoch_670.ckpt
Are 0002 and 0001 used differently? If not can you provide the required documentation?
I tested the method you provided, but there are still bugs. The details are as follows
ns-train nsg-kitti-car-depth-recon --data /home//mars/data/kitti/training/image_02/0002
──────────────────────────────────────────────────────── Config ────────────────────────────────────────────────────────
TrainerConfig(
_target=<class 'nerfstudio.engine.trainer.Trainer'>,
output_dir=PosixPath('outputs'),
method_name='nsg-kitti-car-depth-recon',
experiment_name=None,
project_name='nerfstudio-project',
timestamp='2023-08-03_104959',
machine=MachineConfig(seed=42, num_gpus=1, num_machines=1, machine_rank=0, dist_url='auto'),
logging=LoggingConfig(
relative_log_dir=PosixPath('.'),
steps_per_log=10,
max_buffer_size=20,
local_writer=LocalWriterConfig(
_target=<class 'nerfstudio.utils.writer.LocalWriter'>,
enable=True,
stats_to_track=(
<EventName.ITER_TRAIN_TIME: 'Train Iter (time)'>,
<EventName.TRAIN_RAYS_PER_SEC: 'Train Rays / Sec'>,
<EventName.CURR_TEST_PSNR: 'Test PSNR'>,
<EventName.VIS_RAYS_PER_SEC: 'Vis Rays / Sec'>,
<EventName.TEST_RAYS_PER_SEC: 'Test Rays / Sec'>,
<EventName.ETA: 'ETA (time)'>
),
max_log_size=10
),
profiler='basic'
),
viewer=ViewerConfig(
relative_log_filename='viewer_log_filename.txt',
websocket_port=None,
websocket_port_default=7007,
websocket_host='0.0.0.0',
num_rays_per_chunk=32768,
max_num_display_images=512,
quit_on_train_completion=False,
image_format='jpeg',
jpeg_quality=90
),
pipeline=NSGPipelineConfig(
_target=<class 'nsg.nsg_pipeline.NSGPipeline'>,
datamanager=NSGkittiDataManagerConfig(
_target=<class 'nsg.data.nsg_datamanager.NSGkittiDataManager'>,
data=PosixPath('/home//mars/data/kitti/training/image_02/0002'),
camera_optimizer=CameraOptimizerConfig(
_target=<class 'nerfstudio.cameras.camera_optimizers.CameraOptimizer'>,
mode='off',
position_noise_std=0.0,
orientation_noise_std=0.0,
optimizer=AdamOptimizerConfig(
_target=<class 'torch.optim.adam.Adam'>,
lr=0.0006,
eps=1e-15,
max_norm=None,
weight_decay=0
),
scheduler=ExponentialDecaySchedulerConfig(
_target=<class 'nerfstudio.engine.schedulers.ExponentialDecayScheduler'>,
lr_pre_warmup=1e-08,
lr_final=None,
warmup_steps=0,
max_steps=10000,
ramp='cosine'
),
param_group='camera_opt'
),
dataparser=NSGkittiDataParserConfig(
_target=<class 'nsg.data.nsg_dataparser.NSGkitti'>,
data=PosixPath('data/kitti/training/image_02/0002'),
scale_factor=1,
scene_scale=1.0,
alpha_color='white',
first_frame=140,
last_frame=224,
use_object_properties=True,
object_setting=0,
obj_opaque=True,
box_scale=1.5,
novel_view='left',
use_obj=True,
render_only=False,
bckg_only=False,
near_plane=0.5,
far_plane=150.0,
dataset_type='kitti',
obj_only=False,
netchunk=65536,
chunk=32768,
max_input_objects=-1,
add_input_rows=-1,
use_car_latents=True,
car_object_latents_path=PosixPath('/home//mars/latents/KITTI-MOT/car-object-latents/latent_cod
es02.pt'),
car_nerf_state_dict_path=PosixPath('/home//mars/latents/KITTI-MOT/car-nerf-state-dict/epoch_67
0.ckpt'),
use_depth=True,
split_setting='reconstruction',
use_semantic=False,
semantic_path=PosixPath('.'),
semantic_mask_classes=[]
),
train_num_rays_per_batch=4096,
train_num_images_to_sample_from=-1,
train_num_times_to_repeat_images=-1,
eval_num_rays_per_batch=4096,
eval_num_images_to_sample_from=-1,
eval_num_times_to_repeat_images=-1,
eval_image_indices=(0,),
camera_res_scale_factor=1.0,
patch_size=1
),
model=SceneGraphModelConfig(
_target=<class 'nsg.models.scene_graph.SceneGraphModel'>,
enable_collider=True,
collider_params={'near_plane': 2.0, 'far_plane': 6.0},
loss_coefficients={'rgb_loss_coarse': 1.0, 'rgb_loss_fine': 1.0},
eval_num_rays_per_chunk=4096,
background_model=NerfactoModelConfig(
_target=<class 'nsg.models.nerfacto.NerfactoModel'>,
enable_collider=True,
collider_params={'near_plane': 2.0, 'far_plane': 6.0},
loss_coefficients={'rgb_loss_coarse': 1.0, 'rgb_loss_fine': 1.0},
eval_num_rays_per_chunk=4096,
near_plane=0.05,
far_plane=150.0,
background_color='black',
hidden_dim=64,
hidden_dim_color=64,
hidden_dim_transient=64,
num_levels=16,
max_res=2048,
log2_hashmap_size=19,
num_coarse_samples=24,
distortion_loss_mult=0.002,
orientation_loss_mult=0.0001,
pred_normal_loss_mult=0.001,
use_average_appearance_embedding=True,
predict_normals=False,
obj_feat_dim=0,
disable_scene_contraction=False,
sampler='proposal',
num_proposal_samples_per_ray=(256, 128),
num_nerf_samples_per_ray=97,
proposal_update_every=5,
proposal_warmup=5000,
num_proposal_iterations=2,
use_same_proposal_network=False,
proposal_net_args_list=[
{'hidden_dim': 16, 'log2_hashmap_size': 17, 'num_levels': 5, 'max_res': 128, 'use_linear': False},
{'hidden_dim': 16, 'log2_hashmap_size': 17, 'num_levels': 5, 'max_res': 256, 'use_linear': False}
],
use_single_jitter=True,
use_proposal_weight_anneal=True,
proposal_weights_anneal_slope=10.0,
proposal_weights_anneal_max_num_iters=1000,
use_gradient_scaling=False
),
object_model_template=CarNeRFModelConfig(
_target=<class 'nsg.models.car_nerf.CarNeRF'>,
enable_collider=True,
collider_params={'near_plane': 2.0, 'far_plane': 6.0},
loss_coefficients={'rgb_loss_coarse': 1.0, 'rgb_loss_fine': 1.0},
eval_num_rays_per_chunk=4096,
num_coarse_samples=32,
num_fine_samples=97,
background_color='black',
optimize_latents=False
),
max_num_obj=-1,
ray_add_input_rows=-1,
near_plane=0.05,
far_plane=1000.0,
background_color='black',
latent_size=256,
orientation_loss_mult=0.0001,
pred_normal_loss_mult=0.001,
predict_normals=False,
object_representation='class-wise',
object_ray_sample_strategy='remove-bg',
object_warmup_steps=1000,
depth_loss_mult=0.01,
semantic_loss_mult=1.0,
mono_depth_loss_mult=0.0,
is_euclidean_depth=False,
depth_sigma=0.05,
should_decay_sigma=False,
starting_depth_sigma=4.0,
sigma_decay_rate=0.9998,
depth_loss_type=<DepthLossType.DS_NERF: 1>,
use_interlevel_loss=True,
interlevel_loss_mult=1.0,
debug_object_pose=False,
use_sky_model=False,
sky_model=SkyModelConfig(
_target=<class 'nsg.models.sky_model.SkyModel'>,
enable_collider=True,
collider_params={'near_plane': 2.0, 'far_plane': 6.0},
loss_coefficients={'rgb_loss_coarse': 1.0, 'rgb_loss_fine': 1.0},
eval_num_rays_per_chunk=4096,
hidden_dim=128,
num_layers=5
)
)
),
optimizers={
'background_model': {
'optimizer': RAdamOptimizerConfig(
_target=<class 'torch.optim.radam.RAdam'>,
lr=0.001,
eps=1e-15,
max_norm=None,
weight_decay=0
),
'scheduler': ExponentialDecaySchedulerConfig(
_target=<class 'nerfstudio.engine.schedulers.ExponentialDecayScheduler'>,
lr_pre_warmup=1e-08,
lr_final=1e-05,
warmup_steps=0,
max_steps=200000,
ramp='cosine'
)
},
'learnable_global': {
'optimizer': RAdamOptimizerConfig(
_target=<class 'torch.optim.radam.RAdam'>,
lr=0.001,
eps=1e-15,
max_norm=None,
weight_decay=0
),
'scheduler': ExponentialDecaySchedulerConfig(
_target=<class 'nerfstudio.engine.schedulers.ExponentialDecayScheduler'>,
lr_pre_warmup=1e-08,
lr_final=1e-05,
warmup_steps=0,
max_steps=200000,
ramp='cosine'
)
},
'object_model': {
'optimizer': RAdamOptimizerConfig(
_target=<class 'torch.optim.radam.RAdam'>,
lr=0.005,
eps=1e-15,
max_norm=None,
weight_decay=0
),
'scheduler': ExponentialDecaySchedulerConfig(
_target=<class 'nerfstudio.engine.schedulers.ExponentialDecayScheduler'>,
lr_pre_warmup=1e-08,
lr_final=1e-05,
warmup_steps=0,
max_steps=200000,
ramp='cosine'
)
}
},
vis='wandb',
data=PosixPath('/home//mars/data/kitti/training/image_02/0002'),
relative_model_dir=PosixPath('nerfstudio_models'),
steps_per_save=2000,
steps_per_eval_batch=500,
steps_per_eval_image=500,
steps_per_eval_all_images=5000,
max_num_iterations=600000,
mixed_precision=False,
use_grad_scaler=True,
save_only_latest_checkpoint=False,
load_dir=None,
load_step=None,
load_config=None,
load_checkpoint=None,
log_gradients=True
)
────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
Saving config to: experiment_config.py:129
outputs/0002/nsg-kitti-car-depth-recon/2023-08-03_104959/config.yml
Saving checkpoints to: trainer.py:138
outputs/0002/nsg-kitti-car-depth-recon/2023-08-03_104959/nerfstudio_models
[10:50:00] [array([3.]), array([6.]), array([7.]), array([8.]), array([14.]), array([15.]), nsg_dataparser.py:843
array([16.]), array([17.]), array([18.]), array([19.]), array([9.])] in this scene.
finished data parsing
[10:50:09] [array([3.]), array([6.]), array([7.]), array([8.]), array([14.]), array([15.]), nsg_dataparser.py:843
array([16.]), array([17.]), array([18.]), array([19.]), array([9.])] in this scene.
finished data parsing
Setting up training dataset...
Caching all 170 images.
Setting up evaluation dataset...
Caching all 42 images.
Warning: FullyFusedMLP is not supported for the selected architecture 70. Falling back to CutlassMLP. For maximum performance, raise the target GPU architecture to 75+.
Warning: FullyFusedMLP is not supported for the selected architecture 70. Falling back to CutlassMLP. For maximum performance, raise the target GPU architecture to 75+.
Warning: FullyFusedMLP is not supported for the selected architecture 70. Falling back to CutlassMLP. For maximum performance, raise the target GPU architecture to 75+.
Warning: FullyFusedMLP is not supported for the selected architecture 70. Falling back to CutlassMLP. For maximum performance, raise the target GPU architecture to 75+.
Warning: FullyFusedMLP is not supported for the selected architecture 70. Falling back to CutlassMLP. For maximum performance, raise the target GPU architecture to 75+.
Warning: FullyFusedMLP is not supported for the selected architecture 70. Falling back to CutlassMLP. For maximum performance, raise the target GPU architecture to 75+.
Warning: FullyFusedMLP is not supported for the selected architecture 70. Falling back to CutlassMLP. For maximum performance, raise the target GPU architecture to 75+.
Warning: FullyFusedMLP is not supported for the selected architecture 70. Falling back to CutlassMLP. For maximum performance, raise the target GPU architecture to 75+.
No Nerfstudio checkpoint to load, so training from scratch.
wandb: Tracking run with wandb version 0.15.4
wandb: W&B syncing is set to offline in this directory.
wandb: Run wandb online or set WANDB_MODE=online to enable cloud syncing.
logging events to: outputs/0002/nsg-kitti-car-depth-recon/2023-08-03_104959
Printing profiling stats, from longest to shortest duration in seconds
Trainer.train_iteration: 4.7364
NSGPipeline.get_train_loss_dict: 4.7356
Traceback (most recent call last):
File "/home//anaconda3/envs//bin/ns-train", line 8, in
sys.exit(entrypoint())
File "/home//mars/nerfstudio/nerfstudio/scripts/train.py", line 262, in entrypoint
main(
File "/home//mars/nerfstudio/nerfstudio/scripts/train.py", line 248, in main
launch(
File "/home//mars/nerfstudio/nerfstudio/scripts/train.py", line 187, in launch
main_func(local_rank=0, world_size=world_size, config=config)
File "/home//mars/nerfstudio/nerfstudio/scripts/train.py", line 102, in train_loop
trainer.train()
File "/home//mars/nerfstudio/nerfstudio/engine/trainer.py", line 242, in train
loss, loss_dict, metrics_dict = self.train_iteration(step)
File "/home//mars/nerfstudio/nerfstudio/utils/profiler.py", line 91, in inner
out = func(args, **kwargs)
File "/home//mars/nerfstudio/nerfstudio/engine/trainer.py", line 446, in train_iteration
_, loss_dict, metrics_dict = self.pipeline.get_train_loss_dict(step=step)
File "/home//mars/nerfstudio/nerfstudio/utils/profiler.py", line 91, in inner
out = func(args, **kwargs)
File "/home//mars/nsg/nsg_pipeline.py", line 134, in get_train_loss_dict
model_outputs = self.model(ray_bundle)
File "/home//anaconda3/envs/SUDS/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
return forward_call(input, **kwargs)
File "/home//mars/nerfstudio/nerfstudio/models/base_model.py", line 140, in forward
return self.get_outputs(ray_bundle)
File "/home//mars/nsg/models/scene_graph.py", line 443, in get_outputs
result = model.inference_without_render(ray_obj)
File "/home/*/mars/nsg/models/car_nerf.py", line 228, in inference_without_render
self.car_latents[int(obj_id)].view(1, -1).to(obj_id.device),
KeyError: 3
Hi! Thanks for your feedback. We are now fixing the bug of latent_codes02.pt, we will update it later.
how many iterations you need to iterate on the KITTI dataset in your experiment, I see that the default is 60W iterations in your code
how many iterations you need to iterate on the KITTI dataset in your experiment, I see that the default is 60W iterations in your code
It's up to you, as you can suspend the training at any time.
Hi! Thanks for your feedback. We are now fixing the bug of
latent_codes02.pt, we will update it later.
Thanks for your reply, When do you plan to update the file of latent_codes02.pt, thanks in advance!
Hi, since this version is currently a pre-release version, we are not planning to update the checkpoints. We are currently working on v1.0 which is coming soon.