-
Notifications
You must be signed in to change notification settings - Fork 442
/
Copy path11b_eval.yaml
90 lines (86 loc) · 3.2 KB
/
11b_eval.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
# Eval config for Llama 3.2 11B Vision Instruct.
#
# Usage:
# oumi evaluate -c configs/recipes/vision/llama3_2_vision/evaluation/11b_eval.yaml
#
# See Also:
# - Documentation: https://oumi.ai/docs/en/latest/user_guides/evaluate/evaluate.html
# - Config class: oumi.core.configs.EvaluationConfig
# - Config source: https://github.com/oumi-ai/oumi/blob/main/src/oumi/core/configs/evaluation_config.py
# - Other eval configs: configs/**/evaluation/
model:
model_name: "meta-llama/Llama-3.2-11B-Vision-Instruct"
model_max_length: 1024
torch_dtype_str: "bfloat16"
attn_implementation: "sdpa"
shard_for_eval: True
generation:
batch_size: 1
tasks:
- evaluation_platform: lm_harness
task_name: "mmmu_val"
# num_samples: 100 # Limit the number of eval samples.
eval_kwargs:
num_fewshot: 5
# To enable individual sub-tasks, uncomment the entries of interest below:
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_accounting"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_agriculture"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_architecture_and_engineering"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_art"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_art_theory"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_basic_medical_science"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_biology"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_chemistry"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_clinical_medicine"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_computer_science"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_design"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_diagnostics_and_laboratory_medicine"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_economics"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_electronics"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_energy_and_power"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_finance"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_geography"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_history"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_literature"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_manage"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_marketing"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_materials"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_math"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_mechanical_engineering"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_music"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_pharmacy"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_physics"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_psychology"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_public_health"
# - evaluation_platform: lm_harness
# task_name: "mmmu_val_sociology"
enable_wandb: True