Skip to content

Commit

Permalink
Replace filter_bias_and_bn by weight_decay_filter to fix and be
Browse files Browse the repository at this point in the history 
compatible with previous bugs.
  • Loading branch information
@sageyou
sageyou committed Dec 27, 2023
1 parent 9c6dc67 commit c28e3e6
Show file tree
Hide file tree
Showing 189 changed files with 96 additions and 232 deletions.
10 changes: 8 additions & 2 deletions config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,8 +182,14 @@ def create_parser():
help='Weight decay (default=1e-6)')
group.add_argument('--use_nesterov', type=str2bool, nargs='?', const=True, default=False,
help='Enables the Nesterov momentum (default=False)')
group.add_argument('--filter_bias_and_bn', type=str2bool, nargs='?', const=True, default=True,
help='Filter Bias and BatchNorm (default=True)')
group.add_argument('--weight_decay_filter', type=str, default="disable",
choices=['disable', 'auto', 'norm_and_bias'],
help='filter parameters from weight_decay. '
'choice: "disable" - No parameters to filter from weight_decay; "auto" - In this case, '
'we do not apply weight decay filtering to any parameters. However, MindSpore currently '
'automatically filters norm parameters from weight decay. It is unclear whether there '
'will be any changes in future versions of MindSpore, so it is recommended to stay updated;'
'"norm_and_bias" - Filter the paramtersof Norm layer and Bias from weight decay')
group.add_argument('--eps', type=float, default=1e-10,
help='Term Added to the Denominator to Improve Numerical Stability (default=1e-10)')

Expand Down
2 changes: 1 addition & 1 deletion configs/bit/bit_resnet101_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ multi_step_decay_milestones: [30, 40, 50, 60, 70, 80, 85]

# optimizer
opt: 'sgd'
filter_bias_and_bn: False
weight_decay_filter: 'auto'
momentum: 0.9
weight_decay: 0.0001
loss_scale: 1024
2 changes: 1 addition & 1 deletion configs/bit/bit_resnet50_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ multi_step_decay_milestones: [30, 40, 50, 60, 70, 80, 85]

# optimizer
opt: 'sgd'
filter_bias_and_bn: False
weight_decay_filter: 'auto'
momentum: 0.9
weight_decay: 0.0001
loss_scale: 1024
2 changes: 1 addition & 1 deletion configs/bit/bit_resnet50x3_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ multi_step_decay_milestones: [30, 40, 50, 60, 70, 80, 85]

# optimizer
opt: 'sgd'
filter_bias_and_bn: False
weight_decay_filter: 'auto'
momentum: 0.9
weight_decay: 0.0001
loss_scale: 1024
1 change: 0 additions & 1 deletion configs/cmt/cmt_small_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@ warmup_epochs: 5

# optimizer
opt: "adamw"
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.05
loss_scale_type: 'dynamic'
Expand Down
1 change: 0 additions & 1 deletion configs/coat/coat_lite_mini_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,5 @@ cycle_decay: 1.0
# optimizer
opt: 'adamw'
weight_decay: 0.025
filter_bias_and_bn: True
loss_scale: 1024
use_nesterov: False
1 change: 0 additions & 1 deletion configs/coat/coat_lite_tiny_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,5 @@ cycle_decay: 1.0
# optimizer
opt: 'adamw'
weight_decay: 0.025
filter_bias_and_bn: True
loss_scale: 1024
use_nesterov: False
1 change: 0 additions & 1 deletion configs/coat/coat_mini_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ epoch_size: 300
# optimizer
opt: 'lion'
weight_decay: 0.15
filter_bias_and_bn: True
loss_scale: 4096
use_nesterov: False
loss_scale_type: dynamic
Expand Down
1 change: 0 additions & 1 deletion configs/coat/coat_tiny_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@ epoch_size: 300
# optimizer
opt: 'lion'
weight_decay: 0.15
filter_bias_and_bn: True
loss_scale: 4096
use_nesterov: False
loss_scale_type: dynamic
Expand Down
1 change: 0 additions & 1 deletion configs/convit/convit_base_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,5 +51,4 @@ decay_epochs: 260
opt: 'adamw'
weight_decay: 0.1
loss_scale: 1024
filter_bias_and_bn: True
use_nesterov: False
1 change: 0 additions & 1 deletion configs/convit/convit_base_plus_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,5 +51,4 @@ decay_epochs: 260
opt: 'adamw'
weight_decay: 0.1
loss_scale: 1024
filter_bias_and_bn: True
use_nesterov: False
1 change: 0 additions & 1 deletion configs/convit/convit_small_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,5 +51,4 @@ decay_epochs: 260
opt: 'adamw'
weight_decay: 0.05
loss_scale: 1024
filter_bias_and_bn: True
use_nesterov: False
1 change: 0 additions & 1 deletion configs/convit/convit_small_plus_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,5 +51,4 @@ decay_epochs: 260
opt: 'adamw'
weight_decay: 0.05
loss_scale: 1024
filter_bias_and_bn: True
use_nesterov: False
1 change: 0 additions & 1 deletion configs/convit/convit_tiny_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,5 +50,4 @@ decay_epochs: 295
opt: 'adamw'
weight_decay: 0.0001
loss_scale: 1024
filter_bias_and_bn: True
use_nesterov: False
1 change: 0 additions & 1 deletion configs/convit/convit_tiny_gpu.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,5 +47,4 @@ decay_epochs: 200
# optimizer
opt: 'adamw'
weight_decay: 0.025
filter_bias_and_bn: True
use_nesterov: False
1 change: 0 additions & 1 deletion configs/convit/convit_tiny_plus_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,5 +50,4 @@ decay_epochs: 260
opt: 'adamw'
weight_decay: 0.0001
loss_scale: 1024
filter_bias_and_bn: True
use_nesterov: False
1 change: 0 additions & 1 deletion configs/convnext/convnext_base_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ warmup_epochs: 20

# optimizer
opt: 'adamw'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.05
loss_scale_type: 'auto'
Expand Down
1 change: 0 additions & 1 deletion configs/convnext/convnext_small_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ warmup_epochs: 20

# optimizer
opt: 'adamw'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.05
loss_scale_type: 'auto'
Expand Down
1 change: 0 additions & 1 deletion configs/convnext/convnext_tiny_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ warmup_epochs: 20

# optimizer
opt: 'adamw'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.05
loss_scale_type: 'dynamic'
Expand Down
1 change: 0 additions & 1 deletion configs/convnextv2/convnextv2_tiny_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@ warmup_epochs: 20

# optimizer
opt: 'adamw'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.049
loss_scale_type: 'auto'
Expand Down
1 change: 0 additions & 1 deletion configs/crossvit/crossvit_15_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@ cycle_decay: 1
# optimizer
opt: 'adamw'
weight_decay: 0.05
filter_bias_and_bn: True
loss_scale: 512
use_nesterov: False
eps: 1e-8
Expand Down
1 change: 0 additions & 1 deletion configs/crossvit/crossvit_18_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ decay_rate: 0.1
# optimizer
opt: 'adamw'
weight_decay: 0.05
filter_bias_and_bn: True
loss_scale: 1024
drop_overflow_update: True
loss_scale_type: 'dynamic'
Expand Down
1 change: 0 additions & 1 deletion configs/crossvit/crossvit_9_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ decay_rate: 0.1
# optimizer
opt: 'adamw'
weight_decay: 0.05
filter_bias_and_bn: True
loss_scale_type: 'dynamic'
drop_overflow_update: True
use_nesterov: False
Expand Down
1 change: 0 additions & 1 deletion configs/densenet/densenet_121_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ decay_epochs: 120

# optimizer
opt: 'momentum'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.0001
loss_scale: 1024
Expand Down
1 change: 0 additions & 1 deletion configs/densenet/densenet_121_gpu.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ decay_epochs: 120

# optimizer
opt: 'momentum'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.0001
loss_scale: 1024
Expand Down
1 change: 0 additions & 1 deletion configs/densenet/densenet_161_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ decay_epochs: 120

# optimizer
opt: 'momentum'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.0001
loss_scale: 1024
Expand Down
1 change: 0 additions & 1 deletion configs/densenet/densenet_161_gpu.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ decay_epochs: 120

# optimizer
opt: 'momentum'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.0001
loss_scale: 1024
Expand Down
1 change: 0 additions & 1 deletion configs/densenet/densenet_169_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ decay_epochs: 120

# optimizer
opt: 'momentum'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.0001
loss_scale: 1024
Expand Down
1 change: 0 additions & 1 deletion configs/densenet/densenet_169_gpu.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ decay_epochs: 120

# optimizer
opt: 'momentum'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.0001
loss_scale: 1024
Expand Down
1 change: 0 additions & 1 deletion configs/densenet/densenet_201_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ decay_epochs: 120

# optimizer
opt: 'momentum'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.0001
loss_scale: 1024
Expand Down
1 change: 0 additions & 1 deletion configs/densenet/densenet_201_gpu.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ decay_epochs: 120

# optimizer
opt: 'momentum'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.0001
loss_scale: 1024
Expand Down
1 change: 0 additions & 1 deletion configs/dpn/dpn107_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ decay_epochs: 200

# optimizer
opt: 'SGD'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.0001
loss_scale: 1024
Expand Down
1 change: 0 additions & 1 deletion configs/dpn/dpn131_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ decay_epochs: 200

# optimizer
opt: 'SGD'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.0001
loss_scale: 1024
Expand Down
1 change: 0 additions & 1 deletion configs/dpn/dpn92_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ decay_epochs: 200

# optimizer
opt: 'SGD'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.0001
loss_scale: 1024
Expand Down
1 change: 0 additions & 1 deletion configs/dpn/dpn98_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ decay_epochs: 200

# optimizer
opt: 'SGD'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.0001
loss_scale: 1024
Expand Down
1 change: 0 additions & 1 deletion configs/edgenext/edgenext_base_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@ decay_epochs: 330

# optimizer
opt: 'adamw'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.05
loss_scale: 1024
Expand Down
1 change: 0 additions & 1 deletion configs/edgenext/edgenext_small_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@ decay_epochs: 330

# optimizer
opt: 'adamw'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.05
loss_scale: 1024
Expand Down
1 change: 0 additions & 1 deletion configs/edgenext/edgenext_x_small_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@ decay_epochs: 330

# optimizer
opt: 'adamw'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.05
loss_scale: 1024
Expand Down
1 change: 0 additions & 1 deletion configs/edgenext/edgenext_xx_small_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@ decay_epochs: 330

# optimizer
opt: 'adamw'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.05
loss_scale: 1024
Expand Down
1 change: 0 additions & 1 deletion configs/efficientnet/efficientnet_b0_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ decay_epochs: 445

# optimizer
opt: 'rmsprop'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 1e-5
loss_scale_type: 'dynamic'
Expand Down
1 change: 0 additions & 1 deletion configs/efficientnet/efficientnet_b1_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ decay_epochs: 430

# optimizer
opt: 'rmsprop'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 1e-5
loss_scale_type: 'dynamic'
Expand Down
1 change: 0 additions & 1 deletion configs/ghostnet/ghostnet_050_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ decay_epochs: 580

# optimizer
opt: "momentum"
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.00002
loss_scale_type: "dynamic"
Expand Down
1 change: 0 additions & 1 deletion configs/ghostnet/ghostnet_100_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ decay_epochs: 580

# optimizer
opt: "momentum"
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.00002
loss_scale_type: "dynamic"
Expand Down
1 change: 0 additions & 1 deletion configs/ghostnet/ghostnet_130_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ decay_epochs: 580

# optimizer
opt: "momentum"
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.00002
loss_scale_type: "dynamic"
Expand Down
1 change: 0 additions & 1 deletion configs/googlenet/googlenet_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ warmup_epochs: 5

# optimizer
opt: 'momentum'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.00004
loss_scale: 1024
Expand Down
1 change: 0 additions & 1 deletion configs/halonet/halonet_50t_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ val_amp_level: 'O2'

# optimizer
opt: 'adamw'
filter_bias_and_bn: True
weight_decay: 0.04
loss_scale: 1024
use_nesterov: False
Expand Down
1 change: 0 additions & 1 deletion configs/hrnet/hrnet_w32_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,4 +52,3 @@ decay_epochs: 280
opt: 'adamw'
weight_decay: 0.05
loss_scale: 1024
filter_bias_and_bn: True
1 change: 0 additions & 1 deletion configs/hrnet/hrnet_w48_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,4 +52,3 @@ decay_epochs: 280
opt: 'adamw'
weight_decay: 0.05
loss_scale: 1024
filter_bias_and_bn: True
1 change: 0 additions & 1 deletion configs/inceptionv3/inception_v3_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ warmup_epochs: 5

# optimizer
opt: 'momentum'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.00004
loss_scale: 1024
Expand Down
1 change: 0 additions & 1 deletion configs/inceptionv4/inception_v4_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ warmup_epochs: 5

# optimizer
opt: 'momentum'
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.00004
loss_scale: 1024
Expand Down
1 change: 0 additions & 1 deletion configs/mixnet/mixnet_l_ascend.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ warmup_epochs: 20

# optimizer
opt: "momentum"
filter_bias_and_bn: True
momentum: 0.9
weight_decay: 0.00002
loss_scale_type: "dynamic"
Expand Down
Loading

0 comments on commit c28e3e6

Please sign in to comment.