training_config.yaml

aws_output_bucket: s3://panda-us-west-2/experiments/Llama-2-13b-chat-hf-code-github-c4_v1.pp8.dp2.0822.aws
data_dir: null
dist_load_data_barrier: false
train_file: /tmp/data-train-code-c4/code/
dev_file: null
test_file: null
model:
  _target_: models.llama.LlamaForConditionalGeneration.from_pretrained
  vocab_size: 79458
  use_peft: false
  gradient_checkpointing: true
  enable_flash_attention: true
  flash_attention_vanilla_torch: true
  pad_token_id: 2
read_tensor:
  _target_: data.collators.zh_instruct.TextDatasetCombineV2_CodeGithub
  extra_data:
    _target_: data.collators.zh_instruct.C4CombinedDataset
    tokenizer: null
    file_path: /tmp/data-train-code-c4/c4/
    file_num: 50
extended_vocab: null
collator:
  _target_: data.collators.flan.CombineCollator
  max_seq_length: 2048
  tokenizer: ${model_name_or_path}
  decoder_only: true
  padding: longest
  padding_side: right
num_workers: 4
prefetch_factor: 2
do_preprocess: false
model_name_or_path: /tmp/Llama-2-13b-chat-hf-code-github-c4/
pretrain: null
exp_name: llama2.13b.Code.Github.C4.combine.v1.0.seq2k.w16.adamw.NA100.0822.aws.ds
exp_notes: null
output_dir: /tmp/${exp_name}
resume: null
do_train: true
evaluate_during_training: false
do_eval: false
eval_sub_path: checkpoint-*
per_gpu_train_batch_size: 2
per_gpu_eval_batch_size: 1
learning_rate: 1.0e-05
gradient_accumulation_steps: 64
weight_decay: 0.01
adam_epsilon: 1.0e-06
adam_betas: (0.9, 0.99)
max_grad_norm: 1.0
num_train_epochs: 1
total_dataset_len: 10000000
max_steps: 0
warmup_proportion: 0
warmup_steps: 0
optimizer: null
use_nvlamb: null
bit_training: null
logging_steps: 1
save_best: false
save_steps: 250
eval_steps: 250
ddp_eval: true
no_cuda: false
seed: 42
local_rank: 0
fp16: true
fp16_opt_level: O1
fp16_bfloat16: true
prediction_cfg:
  metric: acc
  measure: 1
  best_checkpoint: null
  best_result: null
eval_forward_fn:
  _target_: general_util.evaluator.DiscriminatorForwardFn
post_process: null
fairscale_config:
  _target_: general_util.fsdp_utils.default_initialize
  fp16: ${fp16}
  move_grads_to_cpu: false
  move_params_to_cpu: false
  flatten_parameters: false
with_lightseq: false
load_lr_scheduler_states: false
ds_cfg:
  train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size}
  gradient_accumulation_steps: ${gradient_accumulation_steps}
  optimizer:
    type: AdamW
    params:
      lr: ${learning_rate}
      betas:
      - 0.9
      - 0.96
      eps: ${adam_epsilon}
      weight_decay: ${weight_decay}
  scheduler:
    type: WarmupDecayLR
    params:
      total_num_steps: 4882
      warmup_max_lr: ${learning_rate}
      warmup_num_steps: 0
      warmup_type: linear
  gradient_clipping: ${max_grad_norm}
  bf16:
    enabled: ${fp16}
  zero_optimization:
    stage: 1
    contiguous_gradients: true
    overlap_comm: true
    reduce_scatter: true
    reduce_bucket_size: 500000000.0
    allgather_bucket_size: 500000000.0
    offload_optimizer:
      device: cpu
      pin_memory: true
  steps_per_print: 1
summary_helper:
  _target_: general_util.tensorboard_helper.WandbWriter
  batch_index_or_keys: null
  outputs_index_or_keys: null
n_gpu: 1
device: cuda:0
train_batch_size: 2
eval_batch_size: null
world_size: 16
world_rank: null