default.json

{
    "dataset": "scifact",
    "sample_num": 10000000,
    "batch_size": 1,
    "eval_batch_size": 8,
    "grad_accum_factor": 2,
    "num_shot": 6,
    "length_norm": 1,
    "mc_loss": 1,
    "unlikely_loss": 1,
    "compute_strategy":"ddp",
    "save_model":true,
    "allow_skip_exp":false ,
    "few_shot_random_seed":0,
    "seed":0,
    "num_steps":1000,
    "max_seq_len":200,
    "save_data_file": true,
    "n_ways":3,
    "eval_before_training": false,
    "load_weight":"pretrained_checkpoints/t03b_ia3_finish.pt",
    "exp_name": "t03b_fever",
    "dataset_offline":"data_dir",
    "origin_model": "bigscience/T0_3B",
    "compute_precision": "fp32",
    "lora_scaling_rank": 1,
    "lora_rank": 0,
    "lora_init_scale": 0.0,
    "lora_modules": ".*SelfAttention|.*EncDecAttention|.*DenseReluDense",
    "lora_layers": "k|v|wi_1.*",
    "trainable_param_names": ".*lora_b.*",
    "model_modifier": "lora",
    "lr": 1e-4
}