为什么执行CUDA_VISIBLE_DEVICES=0,1,2,3 tools/dist_train.sh configs/dota/orientedrepoints_r101_demo.py 4 不报错，执行CUDA_VISIBLE_DEVICES=0,1,2,3 tools/dist_train.sh configs/dota/orientedrepoints_r50_demo.py 4就报下边的错误？

Question

为什么执行CUDA_VISIBLE_DEVICES=0,1,2,3 tools/dist_train.sh configs/dota/orientedrepoints_r101_demo.py 4 不报错，执行CUDA_VISIBLE_DEVICES=0,1,2,3 tools/dist_train.sh configs/dota/orientedrepoints_r50_demo.py 4就报下边的错误？

Opened this issue a year ago · 0 comments

Traceback (most recent call last):
File "tools/train.py", line 154, in
main()
File "tools/train.py", line 127, in main
cfg.model, train_cfg=cfg.train_cfg, test_cfg=cfg.test_cfg)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/builder.py", line 43, in build_detector
return build(cfg, DETECTORS, dict(train_cfg=train_cfg, test_cfg=test_cfg))
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/builder.py", line 15, in build
return build_from_cfg(cfg, registry, default_args)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/utils/registry.py", line 79, in build_from_cfg
return obj_cls(**args)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/detectors/orientedreppoints_detector.py", line 22, in init
pretrained)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/detectors/single_stage.py", line 28, in init
self.bbox_head = builder.build_head(bbox_head)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/builder.py", line 35, in build_head
return build(cfg, HEADS)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/builder.py", line 15, in build
return build_from_cfg(cfg, registry, default_args)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/utils/registry.py", line 79, in build_from_cfg
return obj_cls(**args)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/anchor_heads/orientedreppoints_head.py", line 89, in init
self._init_layers()
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/anchor_heads/orientedreppoints_head.py", line 120, in init_layers
self.reppoints_cls_out = nn.Conv2d(self.point_feat_channels, self.cls_out_channels, 1, 1, 0)
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/modules/conv.py", line 332, in init
False, pair(0), groups, bias, padding_mode)
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/modules/conv.py", line 46, in init
self.reset_parameters()
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/modules/conv.py", line 49, in reset_parameters
init.kaiming_uniform(self.weight, a=math.sqrt(5))
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/init.py", line 310, in kaiming_uniform
fan = _calculate_correct_fan(tensor, mode)
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/init.py", line 279, in _calculate_correct_fan
fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/init.py", line 210, in _calculate_fan_in_and_fan_out
if tensor[0][0].numel() > 2:
IndexError: index 0 is out of bounds for dimension 0 with size 0
Traceback (most recent call last):
File "tools/train.py", line 154, in
main()
File "tools/train.py", line 127, in main
cfg.model, train_cfg=cfg.train_cfg, test_cfg=cfg.test_cfg)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/builder.py", line 43, in build_detector
return build(cfg, DETECTORS, dict(train_cfg=train_cfg, test_cfg=test_cfg))
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/builder.py", line 15, in build
return build_from_cfg(cfg, registry, default_args)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/utils/registry.py", line 79, in build_from_cfg
return obj_cls(**args)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/detectors/orientedreppoints_detector.py", line 22, in init
pretrained)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/detectors/single_stage.py", line 28, in init
self.bbox_head = builder.build_head(bbox_head)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/builder.py", line 35, in build_head
return build(cfg, HEADS)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/builder.py", line 15, in build
return build_from_cfg(cfg, registry, default_args)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/utils/registry.py", line 79, in build_from_cfg
return obj_cls(**args)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/anchor_heads/orientedreppoints_head.py", line 89, in init
self._init_layers()
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/anchor_heads/orientedreppoints_head.py", line 120, in init_layers
self.reppoints_cls_out = nn.Conv2d(self.point_feat_channels, self.cls_out_channels, 1, 1, 0)
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/modules/conv.py", line 332, in init
False, pair(0), groups, bias, padding_mode)
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/modules/conv.py", line 46, in init
self.reset_parameters()
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/modules/conv.py", line 49, in reset_parameters
init.kaiming_uniform(self.weight, a=math.sqrt(5))
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/init.py", line 310, in kaiming_uniform
fan = _calculate_correct_fan(tensor, mode)
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/init.py", line 279, in _calculate_correct_fan
fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/init.py", line 210, in _calculate_fan_in_and_fan_out
if tensor[0][0].numel() > 2:
IndexError: index 0 is out of bounds for dimension 0 with size 0
Traceback (most recent call last):
File "tools/train.py", line 154, in
main()
File "tools/train.py", line 127, in main
cfg.model, train_cfg=cfg.train_cfg, test_cfg=cfg.test_cfg)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/builder.py", line 43, in build_detector
return build(cfg, DETECTORS, dict(train_cfg=train_cfg, test_cfg=test_cfg))
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/builder.py", line 15, in build
return build_from_cfg(cfg, registry, default_args)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/utils/registry.py", line 79, in build_from_cfg
return obj_cls(**args)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/detectors/orientedreppoints_detector.py", line 22, in init
pretrained)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/detectors/single_stage.py", line 28, in init
self.bbox_head = builder.build_head(bbox_head)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/builder.py", line 35, in build_head
return build(cfg, HEADS)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/builder.py", line 15, in build
return build_from_cfg(cfg, registry, default_args)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/utils/registry.py", line 79, in build_from_cfg
return obj_cls(**args)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/anchor_heads/orientedreppoints_head.py", line 89, in init
self._init_layers()
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/anchor_heads/orientedreppoints_head.py", line 120, in init_layers
self.reppoints_cls_out = nn.Conv2d(self.point_feat_channels, self.cls_out_channels, 1, 1, 0)
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/modules/conv.py", line 332, in init
False, pair(0), groups, bias, padding_mode)
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/modules/conv.py", line 46, in init
self.reset_parameters()
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/modules/conv.py", line 49, in reset_parameters
init.kaiming_uniform(self.weight, a=math.sqrt(5))
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/init.py", line 310, in kaiming_uniform
fan = _calculate_correct_fan(tensor, mode)
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/init.py", line 279, in _calculate_correct_fan
fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/init.py", line 210, in _calculate_fan_in_and_fan_out
if tensor[0][0].numel() > 2:
IndexError: index 0 is out of bounds for dimension 0 with size 0
Traceback (most recent call last):
File "tools/train.py", line 154, in
main()
File "tools/train.py", line 127, in main
cfg.model, train_cfg=cfg.train_cfg, test_cfg=cfg.test_cfg)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/builder.py", line 43, in build_detector
return build(cfg, DETECTORS, dict(train_cfg=train_cfg, test_cfg=test_cfg))
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/builder.py", line 15, in build
return build_from_cfg(cfg, registry, default_args)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/utils/registry.py", line 79, in build_from_cfg
return obj_cls(**args)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/detectors/orientedreppoints_detector.py", line 22, in init
pretrained)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/detectors/single_stage.py", line 28, in init
self.bbox_head = builder.build_head(bbox_head)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/builder.py", line 35, in build_head
return build(cfg, HEADS)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/builder.py", line 15, in build
return build_from_cfg(cfg, registry, default_args)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/utils/registry.py", line 79, in build_from_cfg
return obj_cls(**args)
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/anchor_heads/orientedreppoints_head.py", line 89, in init
self._init_layers()
File "/media/wit-ai515-1080ti-02/data-6T/lxj/OrientedRepPoints-main/mmdet/models/anchor_heads/orientedreppoints_head.py", line 120, in init_layers
self.reppoints_cls_out = nn.Conv2d(self.point_feat_channels, self.cls_out_channels, 1, 1, 0)
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/modules/conv.py", line 332, in init
False, pair(0), groups, bias, padding_mode)
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/modules/conv.py", line 46, in init
self.reset_parameters()
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/modules/conv.py", line 49, in reset_parameters
init.kaiming_uniform(self.weight, a=math.sqrt(5))
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/init.py", line 310, in kaiming_uniform
fan = _calculate_correct_fan(tensor, mode)
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/init.py", line 279, in _calculate_correct_fan
fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/nn/init.py", line 210, in _calculate_fan_in_and_fan_out
if tensor[0][0].numel() > 2:
IndexError: index 0 is out of bounds for dimension 0 with size 0
Traceback (most recent call last):
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/runpy.py", line 193, in _run_module_as_main
"main", mod_spec)
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/distributed/launch.py", line 263, in
main()
File "/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/lib/python3.7/site-packages/torch/distributed/launch.py", line 259, in main
cmd=cmd)
subprocess.CalledProcessError: Command '['/home/wit-ai515-1080ti-02/.conda/envs/orientedreppoints/bin/python', '-u', 'tools/train.py', '--local_rank=3', 'configs/dota/orientedrepoints_r50_demo.py', '--launcher', 'pytorch']' returned non-zero exit status 1.