Merge pull request #63 from tanganke/develop

merge develop into main

config/method/classification/clip_continual_finetune.yaml

@@ -0,0 +1,28 @@
+_target_: fusion_bench.method.ContinualImageClassificationFineTuningForCLIP
+
+seed: 42
+# shuffle the order of the tasks
+shuffle_order: true
+learning_rate: 1e-5
+weight_decay: 0
+# number of training steps on each task
+num_steps: 4000
+batch_size: 128
+num_workers: 16
+save_interval: 500
+# if `state_dict_load_path` is not null, the training will be resumed from the state_dict_path
+state_dict_load_path: null
+# if `state_dict_save_path` is not null, the state_dict will be saved to the path after training
+state_dict_save_path: null
+# if `skip_training` is true, use with `state_dict_load_path` to skip training and only evaluate
+skip_training: false
+# === LoRA ===
+use_lora: false
+lora_config:
+  r: 16
+  lora_alpha: 32
+  target_modules:
+    - q_proj
+    - v_proj
+  lora_dropout: 0.1
+  bias: none

config/method/classification/clip_finetune.yaml

@@ -0,0 +1,26 @@
+name: clip_finetune
+seed: 42
+learning_rate: 1e-5
+weight_decay: 0
+num_steps: 4000
+batch_size: 128
+num_workers: 16
+save_interval: 500
+# if `state_dict_load_path` is not null, the training will be resumed from the state_dict_path
+state_dict_load_path: null
+# if `state_dict_save_path` is not null, the state_dict will be saved to the path after training
+state_dict_save_path: null
+# if `skip_training` is true, use with `state_dict_load_path` to skip training and only evaluate
+skip_training: false
+# === LoRA ===
+use_lora: false
+lora_config:
+  r: 16
+  lora_alpha: 32
+  target_modules:
+    - q_proj
+    - v_proj
+  lora_dropout: 0.1
+  bias: none
+# === L-LoRA ===
+use_l_lora: false

config/method/clip_finetune.yaml

@@ -3,8 +3,8 @@ seed: 42
 learning_rate: 1e-5
 weight_decay: 0
 num_steps: 4000
-batch_size: 64
-num_workers: 8
+batch_size: 128
+num_workers: 16
 save_interval: 500
 # if `state_dict_load_path` is not null, the training will be resumed from the state_dict_path
 state_dict_load_path: null

config/method/opcm/opcm.yaml

@@ -0,0 +1,12 @@
+_target_: fusion_bench.method.opcm.opcm.OPCMForCLIP
+
+# shuffle the order of the models
+shuffle_order: true
+# the scaling factor for the SVD projection
+alpha: 0.5
+# the random seed to use
+seed: null
+# save the merged model on every step
+save_on_every_step: true
+# evaluate the merged model on every step
+evaluate_on_every_step: true

config/method/opcm/task_arithmetic.yaml

@@ -0,0 +1,12 @@
+_target_: fusion_bench.method.opcm.task_arithmetic.ContinualTaskArithmeticForCLIP
+
+scaling_factor: 0.3
+
+# shuffle the order of the models
+shuffle_order: true
+# the random seed to use
+seed: null
+# save the merged model on every step
+save_on_every_step: true
+# evaluate the merged model on every step
+evaluate_on_every_step: true

config/method/opcm/ties_merging.yaml

@@ -0,0 +1,18 @@
+_target_: fusion_bench.method.opcm.ties_merging.ContinualTiesMergingForCLIP
+
+# Scaling factor $\lambda$
+scaling_factor: 0.5
+threshold: 20
+# List of keys to remove from the state dict, default is empty
+remove_keys: []
+# Function to merge the models, default is sum. Options are 'sum', 'mean', and 'max'
+merge_func: sum
+
+# shuffle the order of the models
+shuffle_order: true
+# the random seed to use
+seed: null
+# save the merged model on every step
+save_on_every_step: true
+# evaluate the merged model on every step
+evaluate_on_every_step: true

config/method/opcm/weight_average.yaml

@@ -0,0 +1,10 @@
+_target_: fusion_bench.method.opcm.weight_average.ContinualWeightAverageForCLIP
+
+# shuffle the order of the models
+shuffle_order: true
+# the random seed to use
+seed: null
+# save the merged model on every step
+save_on_every_step: true
+# evaluate the merged model on every step
+evaluate_on_every_step: true

config/modelpool/CLIPVisionModelPool/clip-vit-base-patch32_two_tasks_control_task.yaml

@@ -0,0 +1,18 @@
+defaults:
+  - _self_
+  - /dataset/image_classification/train@train_datasets:
+      - tiny-imagenet
+
+_target_: fusion_bench.modelpool.CLIPVisionModelPool
+_recursive_: false
+
+models:
+  _pretrained_: openai/clip-vit-base-patch32
+  model_1: tanganke/clip-vit-base-patch32_sun397
+  model_2: tanganke/clip-vit-base-patch32_stanford-cars
+
+train_datasets: ???
+
+processor:
+  _target_: transformers.CLIPProcessor.from_pretrained
+  pretrained_model_name_or_path: openai/clip-vit-base-patch32

docs/README.md

@@ -172,7 +172,7 @@ fusion_bench \
     # (2)
     method=simple_averaging  \
     # (3)
-    modelpool=clip-vit-base-patch32_robustness_corrupted \
+    modelpool=CLIPVisionModelPool/clip-vit-base-patch32_robustness_corrupted \
     # (4)
     taskpool=clip-vit-base-patch32_robustness_corrupted
 ```
@@ -184,7 +184,7 @@ fusion_bench \
 2. The `method` option specifies the fusion algorithm to be used. In this case, we use the simple averaging algorithm.
 3. Here we specify the model pool to be used. 
     The model pool is responsible for managing the loading, preprocessing, and saving of the models.
-    By pass option `modelpool=clip-vit-base-patch32_robustness_corrupted`, the program instantiate a modelpool object that manages 4 task-specific CLIP-ViT-B/32 models that are fine-tuned on Stanford Cars, EuroSAT, RESISC45, and GTSRB datasets.
+    By pass option `modelpool=CLIPVisionModelPool/clip-vit-base-patch32_robustness_corrupted`, the program instantiate a modelpool object that manages 4 task-specific CLIP-ViT-B/32 models that are fine-tuned on Stanford Cars, EuroSAT, RESISC45, and GTSRB datasets.
 4. Here we specify the task pool to be used. 
     The task pool is responsible for managing the evaluation datasets and metrics.
     By pass option `taskpool=clip-vit-base-patch32_robustness_corrupted`, the program instantiate a taskpool object that manages 4 tasks with data corrupted by Gaussian noise.

examples/clip_finetune/.gitignore

@@ -0,0 +1 @@
+/tanganke/

examples/clip_finetune/clip_finetune.sh

@@ -1,63 +1,105 @@
 #!/bin/bash
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
 # pre-trained model path
-MODEL=/data0/users/tanganke/data/huggingface_models/openai/clip-vit-base-patch16
-MODEL_SHORT_NAME=ViT-B-16
+MODEL=openai/clip-vit-base-patch32
+MODEL_SHORT_NAME=ViT-B-32
 # TASKS: sun397 stanford_cars resisc45 eurosat svhn gtsrb mnist dtd
+TASK=kmnist
 
 # Full fine-tune CLIP-ViT-B/16:
-function full_finetune(){   
-    for TASK in sun397 stanford_cars resisc45 eurosat svhn gtsrb mnist dtd; do
+function full_finetune() {
     fusion_bench \
         --config-dir ${SCRIPT_DIR}/config \
-        method=clip_finetune \
-        method.num_steps=2000 \
+        method=classification/clip_finetune \
+        method.num_steps=4000 \
         method.learning_rate=1e-5 \
         modelpool=clip-finetune_${TASK} \
-        modelpool.models.0.path=${MODEL} \
-        taskpool=clip-vit-classification_TA8 \
-        taskpool.clip_model=${MODEL} \
+        modelpool.base_model=${MODEL} \
         fabric.loggers.root_dir=outputs/${MODEL_SHORT_NAME}/full_finetune \
-        fabric.loggers.name=${TASK} \
-        report_save_path=outputs/${MODEL_SHORT_NAME}/full_finetune_${TASK}.json
-    done
+        fabric.loggers.name=${TASK}
 }
 
-function lora_finetune(){
-    for TASK in sun397 stanford_cars resisc45 eurosat svhn gtsrb mnist dtd; do
+function lora_finetune() {
     fusion_bench \
         --config-dir ${SCRIPT_DIR}/config \
-        method=clip_finetune \
-        method.num_steps=2000 \
+        method=classification/clip_finetune \
+        method.num_steps=4000 \
         method.learning_rate=1e-5 \
         method.use_lora=true \
         modelpool=clip-finetune_${TASK} \
         modelpool.models.0.path=${MODEL} \
-        taskpool=clip-vit-classification_TA8 \
-        taskpool.clip_model=${MODEL} \
         fabric.loggers.root_dir=outputs/${MODEL_SHORT_NAME}/lora_finetune \
-        fabric.loggers.name=${TASK} \
-        report_save_path=outputs/${MODEL_SHORT_NAME}/lora_finetune_${TASK}.json
-    done
+        fabric.loggers.name=${TASK}
 }
 
-function l_lora_finetune(){
-    for TASK in sun397 stanford_cars resisc45 eurosat svhn gtsrb mnist dtd; do
+function l_lora_finetune() {
     fusion_bench \
         --config-dir ${SCRIPT_DIR}/config \
-        method=clip_finetune \
-        method.num_steps=2000 \
+        method=classification/clip_finetune \
+        method.num_steps=4000 \
         method.learning_rate=1e-5 \
         method.use_lora=true \
         method.use_l_lora=true \
         modelpool=clip-finetune_${TASK} \
         modelpool.models.0.path=${MODEL} \
-        taskpool=clip-vit-classification_TA8 \
-        taskpool.clip_model=${MODEL} \
         fabric.loggers.root_dir=outputs/${MODEL_SHORT_NAME}/l_lora_finetune \
-        fabric.loggers.name=${TASK} \
-        report_save_path=outputs/${MODEL_SHORT_NAME}/l_lora_finetune_${TASK}.json
-    done
+        fabric.loggers.name=${TASK}
 }
 
 full_finetune
+
+
+function continual_full_finetune_8_tasks() {
+    fusion_bench \
+        --config-dir ${SCRIPT_DIR}/config \
+        fabric.devices=4 \
+        fabric.loggers.root_dir=outputs/${MODEL_SHORT_NAME}/continual_full_finetune \
+        fabric.loggers.name=TA8 \
+        method=classification/clip_continual_finetune \
+        method.num_steps=4000 \
+        method.batch_size=32 \
+        method.save_interval=2000 \
+        method.learning_rate=1e-5 \
+        method.shuffle_order=true \
+        modelpool=clip-finetune_TA8 \
+        modelpool.base_model=${MODEL}
+}
+
+function continual_full_finetune_14_tasks() {
+    fusion_bench \
+        --config-dir ${SCRIPT_DIR}/config \
+        fabric.devices=4 \
+        fabric.loggers.root_dir=outputs/${MODEL_SHORT_NAME}/continual_full_finetune \
+        fabric.loggers.name=TALL14 \
+        method=classification/clip_continual_finetune \
+        method.num_steps=4000 \
+        method.batch_size=32 \
+        method.save_interval=2000 \
+        method.learning_rate=1e-5 \
+        method.shuffle_order=true \
+        modelpool=clip-finetune_TALL14 \
+        modelpool.base_model=${MODEL}
+}
+
+
+function continual_full_finetune_20_tasks() {
+    fusion_bench \
+        --config-dir ${SCRIPT_DIR}/config \
+        fabric.devices=4 \
+        fabric.loggers.root_dir=outputs/${MODEL_SHORT_NAME}/continual_full_finetune \
+        fabric.loggers.name=TALL20 \
+        method=classification/clip_continual_finetune \
+        method.num_steps=4000 \
+        method.batch_size=32 \
+        method.save_interval=2000 \
+        method.learning_rate=1e-5 \
+        method.shuffle_order=true \
+        modelpool=clip-finetune_TALL20 \
+        modelpool.base_model=${MODEL}
+}
+
+for version in 0 1 2 3 4 5 6 7 8 9; do
+    continual_full_finetune_8_tasks
+    continual_full_finetune_14_tasks
+    continual_full_finetune_20_tasks
+done

examples/clip_finetune/clip_finetune_B16.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+# pre-trained model path
+MODEL=openai/clip-vit-base-patch16
+MODEL_SHORT_NAME=ViT-B-16
+# TASKS: sun397 stanford_cars resisc45 eurosat svhn gtsrb mnist dtd
+
+# Full fine-tune CLIP-ViT-B/16:
+function full_finetune() {
+    fusion_bench \
+        --config-dir ${SCRIPT_DIR}/config \
+        fabric.devices=8 \
+        method=classification/clip_finetune \
+        method.num_steps=4000 \
+        method.batch_size=16 \
+        method.save_interval=2000 \
+        method.learning_rate=1e-5 \
+        modelpool=clip-finetune_${TASK} \
+        modelpool.base_model=${MODEL} \
+        fabric.loggers.root_dir=outputs/${MODEL_SHORT_NAME}/full_finetune \
+        fabric.loggers.name=${TASK}
+}
+
+function continual_full_finetune_8_tasks() {
+    fusion_bench \
+        --config-dir ${SCRIPT_DIR}/config \
+        fabric.devices=4 \
+        fabric.loggers.root_dir=outputs/${MODEL_SHORT_NAME}/continual_full_finetune \
+        fabric.loggers.name=TA8 \
+        method=classification/clip_continual_finetune \
+        method.num_steps=4000 \
+        method.batch_size=32 \
+        method.save_interval=2000 \
+        method.learning_rate=1e-5 \
+        method.shuffle_order=true \
+        modelpool=clip-finetune_TA8 \
+        modelpool.base_model=${MODEL}
+}
+
+function continual_full_finetune_14_tasks() {
+    fusion_bench \
+        --config-dir ${SCRIPT_DIR}/config \
+        fabric.devices=4 \
+        fabric.loggers.root_dir=outputs/${MODEL_SHORT_NAME}/continual_full_finetune \
+        fabric.loggers.name=TALL14 \
+        method=classification/clip_continual_finetune \
+        method.num_steps=4000 \
+        method.batch_size=32 \
+        method.save_interval=2000 \
+        method.learning_rate=1e-5 \
+        method.shuffle_order=true \
+        modelpool=clip-finetune_TALL14 \
+        modelpool.base_model=${MODEL}
+}
+
+
+function continual_full_finetune_20_tasks() {
+    fusion_bench \
+        --config-dir ${SCRIPT_DIR}/config \
+        fabric.devices=4 \
+        fabric.loggers.root_dir=outputs/${MODEL_SHORT_NAME}/continual_full_finetune \
+        fabric.loggers.name=TALL20 \
+        method=classification/clip_continual_finetune \
+        method.num_steps=4000 \
+        method.batch_size=32 \
+        method.save_interval=2000 \
+        method.learning_rate=1e-5 \
+        method.shuffle_order=true \
+        modelpool=clip-finetune_TALL20 \
+        modelpool.base_model=${MODEL}
+}
+
+for version in 0 1 2 3 4 5 6 7 8 9; do
+    continual_full_finetune_8_tasks
+    continual_full_finetune_14_tasks
+    continual_full_finetune_20_tasks
+done
+
+for TASK in oxford_flowers102 pcam fer2013 oxford-iiit-pet stl10 food101 fashion_mnist emnist_letters kmnist rendered-sst2; do
+    full_finetune
+done