Merge pull request #542 from xiaofengShi/master

add aquila sql

examples/Aquila/Aquila-sql/Aquila-pretrain-4n8g.yaml

@@ -0,0 +1,29 @@
+# comments
+epochs: 3
+batch_size: 8
+gradient_accumulation_steps: 1
+# lr: 1.0e-7
+# lr: 1.0e-6
+lr: 2.0e-7
+warm_up: 0.01
+save_interval: 2000
+eps: 1.e-8
+bmt_lr_decay_style: "cosine"
+bmt_cpu_offload: False
+skip_iters: 0
+
+# sft
+bmt_pre_load: True
+
+fp16: True
+# flash
+enable_flashattn: True
+enable_flash_attn_models: True
+
+# dataset
+pretrain_data: True
+enable_pretrain_dataset_prefix: "./pretrain_data/opensource-sql-data-weighted-pretrain_add_sql_bw_add_chase-1turn-add-bw_text_document"
+
+# save_optim: True
+save_optim: True
+save_rng: True

examples/Aquila/Aquila-sql/Aquila-sft-sql.yaml

@@ -0,0 +1,31 @@
+# comments
+epochs: 3
+batch_size: 4
+save_interval: 2000
+fp16: True
+
+# lr
+lr: 9.65e-6
+warm_up: 0.1
+weight_decay: 0.1
+adam_beta1: 0.9
+adam_beta2: 0.95
+gradient_accumulation_steps: 1
+
+# bmtrain
+bmt_lr_decay_style: "linear"
+bmt_cpu_offload: False
+bmt_pre_load: True
+
+# flash-attn
+enable_flash_attn_models: True
+
+# sft_dataset
+pretrain_data: False
+enable_sft_conversations_dataset_v2: True
+enable_sft_dataset_dir: "./data"
+enable_sft_dataset_file: "sql_generate_from_spider_wikisql_train_weighted.jsonl"
+
+# save_optim: True
+save_optim: False
+save_rng: False