-
Notifications
You must be signed in to change notification settings - Fork 14
/
ret_msrvtt.yaml
85 lines (78 loc) · 2.84 KB
/
ret_msrvtt.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
# (1) each train_file (json) contains a python list where each item is {'image': img_path, 'caption': text or list_of_text }
# (2) this also accepts a two-element sublist, where the 1st is the anno json file as above (1), the 2nd is image_root, it will be joined with the `image` (image path)
data_root: ${oc.env:SL_DATA_DIR}/videos_images
anno_root_downstream: ${oc.env:SL_DATA_DIR}/anno_downstream
train_file: ['${anno_root_downstream}/msrvtt_ret_train7k.json', '${data_root}/msrvtt_2fps_224', video]
test_types: [msrvtt_1k_test, ]
test_file:
msrvtt_1k_test: ['${anno_root_downstream}/msrvtt_ret_test1k.json', '${data_root}/msrvtt_2fps_224', video]
stop_key: None # used to choose the best ckpt. If None, save the last.
text_encoder: bert-base-uncased
bert_config: configs/config_bert.json
vit_type: beit # items in ${vit_zoo}
vit_zoo: # from huggingface
beit: microsoft/beit-base-patch16-224-pt22k-ft22k
vit_name_or_pretrained_path: ${vit_zoo[${vit_type}]}
temporal_vision_encoder:
enable: False
num_layers: 2
update_pooler_embed: False
add_temporal_embed: False # whether to add temporal embed to encoded frames
image_res: 224
embed_dim: 256
video_input: # input
num_frames: 1
reader: decord # one of [decord, av]
sample_type: rand
num_frames_test: 4 # num_frames during inference/test
sample_type_test: middle
max_txt_l: 32
batch_size:
image: 160
video: 32
batch_size_test:
image: 128
video: 32
k_test: 128
temp: 0.07
loss_weight:
itc: 1.0
itm: 1.0
itm_hard_neg: True
optimizer:
opt: adamW
lr: 1e-5
opt_betas: [0.9, 0.999] # default
weight_decay: 0.02
max_grad_norm: -1 # requires a positive float, use -1 to disable
different_lr: # use a different lr for some modules, e.g., larger lr for new modules
enable: False
module_names: []
lr: 1e-3
scheduler:
sched: cosine
epochs: 5
min_lr_multi: 0.1 # min_lr will be `optimizer.lr * min_lr_multi`
warmup_epochs: 0 # float
output_dir: None # output dir
resume: False # if True, load optimizer and scheduler states as well
pretrained_path: None # path to pretrained model weights, for resume only?
evaluate: False
# `eval_frame_ensemble': how do we aggregate scores if `video_input.num_frames_test' > `video_input.num_frames'
# `concat': concat frames before input to multi-modal encoder, i.e., early fusion
# `mean', `max', `lse': mean/max/lse-pool scores after multi-modal encoder, i.e., late fusion, as in ClipBERT
eval_frame_ensemble: concat # [concat, max, mean, lse]
eval_x_only: False
eval_offload: False # offload image gpu tensors to cpu to save memory, when meet OOM error.
device: cuda
seed: 42
log_freq: 100
dist_url: env://
distributed: True
fp16: True
debug: False
num_workers: 16
wandb:
enable: False
entity: None # username or teamname to store the runs, see https://docs.wandb.ai/ref/python/init
project: msrvtt_ret # setup in your command line