-
Notifications
You must be signed in to change notification settings - Fork 0
/
data.sh
60 lines (47 loc) · 2.03 KB
/
data.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
python tools/preprocess_data.py \
--input "/data/xunjian_yin/mycode/split_files_03/part_*" \
--output-prefix /data/xunjian_yin/mycode/DCLM/dclm-b1/dclm_reverse/dclm_03 \
--tokenizer-model neo/tokenizer.model \
--tokenizer-type SentencePieceTokenizer \
--keep-sequential-samples \
--workers 64 \
--partitions 32 \
--append-eod
# python tools/preprocess_data.py \
# --input "/data/datasets/yxj/code_code.001*" \
# --output-prefix /data/datasets/yxj/rr_code_code.001 \
# --tokenizer-model neo/tokenizer.model \
# --tokenizer-type SentencePieceTokenizer \
# --keep-sequential-samples \
# --workers 100 \
# --partitions 10 \
# --append-eod
# python tools/preprocess_data.py \
# --input "/data/public_models/huggingface/yxj/paper_math.000*" \
# --output-prefix /data/public_models/huggingface/matrix/rr_paper_math.000 \
# --tokenizer-model neo/tokenizer.model \
# --tokenizer-type SentencePieceTokenizer \
# --keep-sequential-samples \
# --workers 99 \
# --partitions 3 \
# --append-eod
# python tools/preprocess_data.py \
# --input "/data/public_models/huggingface/yxj/wiki_*" \
# --output-prefix /data/public_models/huggingface/matrix/rr_wiki \
# --tokenizer-model neo/tokenizer.model \
# --tokenizer-type SentencePieceTokenizer \
# --keep-sequential-samples \
# --workers 100 \
# --partitions 2 \
# --append-eod
# python tools/preprocess_data.py \
# --input "/data/public_models/huggingface/yxj/book_reviews.0000.jsonl" \
# --output-prefix /data/datasets/yxj/ \
# --tokenizer-model neo/tokenizer.model \
# --tokenizer-type SentencePieceTokenizer \
# --keep-sequential-samples \
# --workers 8 \
# --partitions 1 \
# --append-eod
# python tools/count_mmap_token.py --mmap_path /data/datasets/yxj/_text_document
# python tools/count_mmap_token.py --mmap_path /data/public_models/huggingface/matrix/tmp/book_all_text_document