[Model] Add aquila 3b configuration file

examples/aquila/conf/train/train_aquila_3b.yaml

@@ -0,0 +1,100 @@
+system:
+  tensor_model_parallel_size: 1
+  pipeline_model_parallel_size: 2
+  disable_bias_linear: True
+  use_flash_attn: True
+  use_distributed_optimizer: True
+  precision:
+    bf16: True
+    initial_loss_scale: 522893
+    min_loss_scale: 1.0
+    attention_softmax_in_fp32: True
+    accumulate_allreduce_grads_in_fp32: True
+  logging:
+    log_interval: 1
+    log_throughput: True
+    tensorboard_log_interval: 1
+    wandb_project: "train-aquila-3B" 
+    wandb_exp_name: "train-test-3B" 
+  checkpoint:
+    load: outputs_llama3/checkpoint_mc
+    ckpt_format: torch
+    save_interval: 2000 
+
+  hetero:
+    enable_hetero: True
+    hetero_use_cpu_communication: False
+    # mesh format [tp1,cp1,ep1,dp1,pp1,(tp2,cp2...)]
+
+    hetero_pipeline_layer_split: [18, 18]
+    hetero_process_meshes: [1, 1, 1, 4, 2]
+    hetero_device_types: ["A800"]
+
+    standalone_embedding_stage: False
+    hetero_current_device_type: "A800"
+
+  # recompute:
+  #   recompute_granularity: "full"
+  #   recompute_method: "uniform"
+  #   recompute_num_layers: 1
+
+  #   ## pp 2 stages
+  #   recompute_granularity_per_stage_micro_batch:
+  #     - [1, 4, 1, 4, 0]
+  #     - [1, 8, 1, 0, 0]
+  #   recompute_method_per_stage_micro_batch:
+  #     - [1, 8, 1, 0, 0]
+  #     - [1, 8, 1, 0, 0]
+  #   recompute_num_layers_per_stage_micro_batch:
+  #     - [1, 8, 16, 0, 0]
+  #     - [1, 0, 16, 8, 0]
+
+model:
+  # use_mcore_models: True # deprecated
+  transformer_impl: transformer_engine
+  num_layers: 36 
+  hidden_size: 2048 
+  num_attention_heads: 16
+  group_query_attention: True
+  num_query_groups: 2
+  seq_length: 4096 
+  max_position_embeddings: 4096 # only for adding position embeddings
+  norm_epsilon: 1e-6
+  use_rotary_position_embeddings: true
+  no_position_embedding: true
+  rotary_base: 1000000
+  swiglu: true
+  multiple_of: 256
+  hidden_dim_multiplier: 2 # ffn_hidden_size 11008
+  normalization: RMSNorm
+  position_embedding_type: rope
+  untie_embeddings_and_output_weights: False
+  init_method_std: 0.02
+  attention_dropout: 0.0
+  hidden_dropout: 0.0
+  weight_decay: 0.1
+  clip_grad: 1.0
+  train_samples: 20480000
+  eval_iters: 0
+  micro_batch_size: 2
+  global_batch_size: 2048
+  seed: 1234
+
+  optimizer:
+    weight_decay: 0.1
+    adam_beta1: 0.9
+    adam_beta2: 0.95
+    lr_scheduler:
+      lr: 2.0e-5
+      min_lr: 2.0e-6
+      lr_warmup_samples: 2000
+      lr_decay_style: cosine 
+
+data:
+  data_path: ${data_path:??}
+  split: 1 
+  tokenizer:
+    tokenizer_type: Qwen2TokenizerFS
+    tokenizer_path: examples/aquila/qwentokenizer
+    vocab_size: 151936
+    make_vocab_size_divisible_by: 64