-
Notifications
You must be signed in to change notification settings - Fork 187
/
eb5659d0-fb6a-49e5-a311-f1f89412f726.txt
6836 lines (6772 loc) · 461 KB
/
eb5659d0-fb6a-49e5-a311-f1f89412f726.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
====================================================================================================
import os
import sys
with open(sys.argv[0]) as f:
code = f.read() # read the code of this file ASAP, for logging
import uuid
import glob
import time
from dataclasses import dataclass
import numpy as np
import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
# -----------------------------------------------------------------------------
# Muon optimizer
def zeropower_via_svd(G, steps=None):
U, S, V = G.svd()
return U @ V.T
@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
"""
Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
zero even beyond the point where the iteration no longer converges all the way to one everywhere
on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
where S' is diagonal with S_{ii}' \sim Uniform(0.5, 1.5), which turns out not to hurt model
performance at all relative to UV^T, where USV^T = G is the SVD.
"""
assert len(G.shape) == 2
a, b, c = (3.4445, -4.7750, 2.0315)
X = G.bfloat16() / (G.norm() + eps) # ensure top singular value <= 1
if G.size(0) > G.size(1):
X = X.T
for _ in range(steps):
A = X @ X.T
B = A @ X
X = a * X + b * B + c * A @ B
if G.size(0) > G.size(1):
X = X.T
return X.to(G.dtype)
zeropower_backends = dict(svd=zeropower_via_svd, newtonschulz5=zeropower_via_newtonschulz5)
class Muon(torch.optim.Optimizer):
"""
Muon: MomentUm Orthogonalized by Newton-schulz
Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
the advantage that it can be stably run in bfloat16 on the GPU.
Some warnings:
- This optimizer assumes that all parameters passed in are 2D.
- It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
parameters; those should all be optimized by a standard method (e.g., AdamW).
- To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
- We believe it is unlikely to work well for training with small batch size.
- We believe it may not work well for finetuning pretrained models, but we haven't tested this.
- We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).
Arguments:
lr: The learning rate used by the internal SGD.
momentum: The momentum used by the internal SGD.
nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
backend: The chosen backend for the orthogonalization step. (recommended: 'newtonschulz5')
backend_steps: The number of iteration steps to use in the backend, if it is iterative.
"""
def __init__(self, params, lr=3e-4, momentum=0.95, nesterov=True, backend='newtonschulz5', backend_steps=5):
defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, backend=backend, backend_steps=backend_steps)
super().__init__(params, defaults)
def step(self):
for group in self.param_groups:
lr = group['lr']
momentum = group['momentum']
zeropower_backend = zeropower_backends[group['backend']]
for p in group['params']:
g = p.grad
if g is None:
continue
state = self.state[p]
if 'momentum_buffer' not in state:
state['momentum_buffer'] = torch.zeros_like(g)
buf = state['momentum_buffer']
buf.mul_(momentum).add_(g)
if group['nesterov']:
g = g.add(buf, alpha=momentum)
if g.size(0) == 3 * g.size(1): # split grouped QKV parameters
g = torch.cat([zeropower_backend(g1, steps=group['backend_steps']) for g1 in g.split(g.size(1))])
scale = g.size(1)**0.5
else:
g = zeropower_backend(g, steps=group['backend_steps'])
scale = max(g.size(0), g.size(1))**0.5 # scale to have update.square().mean() == 1
p.data.add_(g, alpha=-lr * scale)
# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model
class Rotary(torch.nn.Module):
def __init__(self, dim, base=10000):
super().__init__()
inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
self.register_buffer("inv_freq", inv_freq)
self.seq_len_cached = None
self.cos_cached = None
self.sin_cached = None
def forward(self, x):
seq_len = x.shape[1]
if seq_len != self.seq_len_cached:
self.seq_len_cached = seq_len
t = torch.arange(seq_len, device=x.device).type_as(self.inv_freq)
freqs = torch.outer(t, self.inv_freq).to(x.device)
self.cos_cached = freqs.cos()
self.sin_cached = freqs.sin()
return self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
def apply_rotary_emb(x, cos, sin):
assert x.ndim == 4 # multihead attention
d = x.shape[3]//2
x1 = x[..., :d]
x2 = x[..., d:]
y1 = x1 * cos + x2 * sin
y2 = x1 * (-sin) + x2 * cos
return torch.cat([y1, y2], 3)
def rmsnorm(x0, eps=1e-6):
x = x0.float()
x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + eps)
return x.type_as(x0)
class CausalSelfAttention(nn.Module):
def __init__(self, config):
super().__init__()
self.n_head = config.n_head
self.n_embd = config.n_embd
self.head_dim = self.n_embd // self.n_head
assert self.n_embd % self.n_head == 0
# key, query, value projections for all heads, but in a batch
self.c_attn = nn.Linear(self.n_embd, 3 * self.n_embd, bias=False)
# output projection
self.c_proj = nn.Linear(self.n_embd, self.n_embd, bias=False)
self.rotary = Rotary(self.head_dim)
def forward(self, x):
B, T, C = x.size() # batch size, sequence length, embedding dimensionality (n_embd)
# calculate query, key, values for all heads in batch and move head forward to be the batch dim
qkv = self.c_attn(x)
q, k, v = qkv.split(self.n_embd, dim=2)
k = k.view(B, T, self.n_head, self.head_dim)
q = q.view(B, T, self.n_head, self.head_dim)
v = v.view(B, T, self.n_head, self.head_dim)
cos, sin = self.rotary(q)
q = apply_rotary_emb(q, cos, sin)
k = apply_rotary_emb(k, cos, sin)
y = F.scaled_dot_product_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), is_causal=True)
y = y.transpose(1, 2).contiguous().view(B, T, C) # re-assemble all head outputs side by side
# output projection
y = self.c_proj(y)
return y
class MLP(nn.Module):
def __init__(self, config):
super().__init__()
self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd, bias=False)
self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd, bias=False)
def forward(self, x):
x = self.c_fc(x)
x = F.gelu(x)
x = self.c_proj(x)
return x
class Block(nn.Module):
def __init__(self, config):
super().__init__()
self.attn = CausalSelfAttention(config)
self.mlp = MLP(config)
self.attn_scale = (1 / (2 * config.n_layer)**0.5)
def forward(self, x):
x = x + self.attn_scale * self.attn(rmsnorm(x))
x = x + self.mlp(rmsnorm(x))
return x
# -----------------------------------------------------------------------------
# The main GPT-2 model
@dataclass
class GPTConfig:
vocab_size : int = 50257
n_layer : int = 12
n_head : int = 12
n_embd : int = 768
class GPT(nn.Module):
def __init__(self, config):
super().__init__()
self.config = config
self.transformer = nn.ModuleDict(dict(
wte = nn.Embedding(config.vocab_size, config.n_embd),
h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
))
self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
self.transformer.wte.weight = self.lm_head.weight # https://paperswithcode.com/method/weight-tying
def forward(self, idx, targets=None, return_logits=True):
b, t = idx.size()
pos = torch.arange(0, t, dtype=torch.long, device=idx.device) # shape (t)
# forward the GPT model itself
x = self.transformer.wte(idx) # token embeddings of shape (b, t, n_embd)
for block in self.transformer.h:
x = block(x)
x = rmsnorm(x)
if targets is not None:
# if we are given some desired targets also calculate the loss
logits = self.lm_head(x)
logits = logits.float() # use tf32/fp32 for logits
loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1)
else:
# inference-time mini-optimization: only forward the lm_head on the very last position
logits = self.lm_head(x[:, [-1], :]) # note: using list [-1] to preserve the time dim
logits = logits.float() # use tf32/fp32 for logits
loss = None
# there are performance reasons why not returning logits is prudent, if not needed
if not return_logits:
logits = None
return logits, loss
# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader
def _peek_data_shard(filename):
# only reads the header, returns header data
with open(filename, "rb") as f:
# first read the header, which is 256 int32 integers (4 bytes each)
header = np.frombuffer(f.read(256*4), dtype=np.int32)
if header[0] != 20240520:
print("ERROR: magic number mismatch in the data .bin file!")
print("---> HINT: Are you passing in a correct file with --input_bin?")
print("---> HINT: Dataset encoding changed recently, re-run data prepro or refer again to README")
print("---> HINT: For example re-run: `python dev/data/tinyshakespeare.py`, then re-try")
exit(1)
assert header[1] == 1, "unsupported version"
ntok = header[2] # number of tokens (claimed)
return ntok # for now just return the number of tokens
def _load_data_shard(filename):
with open(filename, "rb") as f:
# first read the header, which is 256 int32 integers (4 bytes each)
header = np.frombuffer(f.read(256*4), dtype=np.int32)
assert header[0] == 20240520, "magic number mismatch in the data .bin file"
assert header[1] == 1, "unsupported version"
ntok = header[2] # number of tokens (claimed)
# the rest of it are tokens, stored as uint16
tokens = np.frombuffer(f.read(), dtype=np.uint16)
assert len(tokens) == ntok, "number of tokens read does not match header?"
return tokens
class DistributedDataLoader:
def __init__(self, filename_pattern, B, T, process_rank, num_processes):
self.process_rank = process_rank
self.num_processes = num_processes
self.B = B
self.T = T
# glob files that match the pattern
self.files = sorted(glob.glob(filename_pattern))
assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"
# load and validate all data shards, count number of tokens in total
ntok_total = 0
for fname in self.files:
shard_ntok = _peek_data_shard(fname)
assert shard_ntok >= num_processes * B * T + 1
ntok_total += int(shard_ntok)
self.ntok_total = ntok_total
# kick things off
self.reset()
def reset(self):
self.current_shard = 0
self.current_position = self.process_rank * self.B * self.T
self.tokens = _load_data_shard(self.files[self.current_shard])
def advance(self): # advance to next data shard
self.current_shard = (self.current_shard + 1) % len(self.files)
self.current_position = self.process_rank * self.B * self.T
self.tokens = _load_data_shard(self.files[self.current_shard])
def next_batch(self):
B = self.B
T = self.T
buf = self.tokens[self.current_position : self.current_position+B*T+1]
buf = torch.tensor(buf.astype(np.int32), dtype=torch.long)
x = (buf[:-1]).view(B, T) # inputs
y = (buf[1:]).view(B, T) # targets
# advance current position and load next shard if necessary
self.current_position += B * T * self.num_processes
if self.current_position + (B * T * self.num_processes + 1) > len(self.tokens):
self.advance()
return x.cuda(), y.cuda()
# -----------------------------------------------------------------------------
# int main
@dataclass
class Hyperparameters:
# data hyperparams
input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
# optimization hyperparams
batch_size : int = 8*64 # batch size, in sequences, across all devices
device_batch_size : int = 64 # batch size, in sequences, per device
sequence_length : int = 1024 # sequence length, in tokens
num_iterations : int = 6200 # number of iterations to run
learning_rate : float = 0.0036
warmup_iters : int = 0
warmdown_iters : int = 1800 # number of iterations of linear warmup/warmdown for triangular or trapezoidal schedule
weight_decay : float = 0
# evaluation and logging hyperparams
val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()
# set up DDP (distributed data parallel). torchrun sets this env variable
assert torch.cuda.is_available()
dist.init_process_group(backend='nccl')
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
device = f'cuda:{ddp_local_rank}'
torch.cuda.set_device(device)
print(f"using device: {device}")
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.
# convenience variables
B, T = args.device_batch_size, args.sequence_length
# calculate the number of steps to take in the val loop.
assert args.val_tokens % (B * T * ddp_world_size) == 0
val_steps = args.val_tokens // (B * T * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (B * ddp_world_size) == 0
train_accumulation_steps = args.batch_size // (B * ddp_world_size)
# load tokens
train_loader = DistributedDataLoader(args.input_bin, B, T, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, B, T, ddp_rank, ddp_world_size)
if master_process:
print(f"Training DataLoader: total number of tokens: {train_loader.ntok_total} across {len(train_loader.files)} files")
print(f"Validation DataLoader: total number of tokens: {val_loader.ntok_total} across {len(val_loader.files)} files")
x, y = train_loader.next_batch()
# init the model from scratch
num_vocab = 50257
model = GPT(GPTConfig(vocab_size=num_vocab, n_layer=12, n_head=12, n_embd=768))
model = model.cuda()
if hasattr(config, "coordinate_descent_tuning"):
config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank])
raw_model = model.module # always contains the "raw" unwrapped model
ctx = torch.amp.autocast(device_type='cuda', dtype=torch.bfloat16)
# init the optimizer(s)
optimizer1 = torch.optim.AdamW(raw_model.lm_head.parameters(), lr=args.learning_rate, betas=(0.9, 0.95),
weight_decay=args.weight_decay, fused=True)
optimizer2 = Muon(raw_model.transformer.h.parameters(), lr=0.1*args.learning_rate, momentum=0.95)
optimizers = [optimizer1, optimizer2]
# learning rate decay scheduler (linear warmup and warmdown)
def get_lr(it):
assert it <= args.num_iterations
# 1) linear warmup for warmup_iters steps
if it < args.warmup_iters:
return (it+1) / args.warmup_iters
# 2) constant lr for a while
elif it < args.num_iterations - args.warmdown_iters:
return 1.0
# 3) linear warmdown
else:
decay_ratio = (args.num_iterations - it) / args.warmdown_iters
return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]
# begin logging
if master_process:
run_id = str(uuid.uuid4())
logdir = 'logs/%s/' % run_id
os.makedirs(logdir, exist_ok=True)
logfile = 'logs/%s.txt' % run_id
# create the log file
with open(logfile, "w") as f:
# begin the log by printing this file (the Python code)
f.write('='*100 + '\n')
f.write(code)
f.write('='*100 + '\n')
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
f.write(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:\n")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
f.write(f'{result.stdout}\n')
f.write('='*100 + '\n')
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.time()
# begin training
train_loader.reset()
for step in range(args.num_iterations + 1):
last_step = (step == args.num_iterations)
# This effectively ignores timing first 10 steps, which are slower for weird reasons.
# Alternately, and slightly more correctly in terms of benchmarking, we could do 10
# steps with dummy data first, and then re-initialize the model and reset the loader.
if step == 10:
training_time_ms = 0
t0 = time.time()
timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val
# once in a while evaluate the validation dataset
if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
# stop the clock
torch.cuda.synchronize()
training_time_ms += 1000 * (time.time() - t0)
# run validation batches
model.eval()
val_loader.reset()
val_loss = 0.0
for _ in range(val_steps):
x_val, y_val = val_loader.next_batch()
with torch.no_grad(): # of course, we'd like to use ctx here too, but that creates a torch.compile error for some reason
_, loss = model(x_val, y_val, return_logits=False)
val_loss += loss
dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
val_loss /= val_steps
# log val loss to console and to logfile
if master_process:
print(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
with open(logfile, "a") as f:
f.write(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms\n')
# start the clock again
torch.cuda.synchronize()
t0 = time.time()
if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
# stop the clock
torch.cuda.synchronize()
training_time_ms += 1000 * (time.time() - t0)
# save the state of the training process
log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
# start the clock again
torch.cuda.synchronize()
t0 = time.time()
# bit confusing: we want to make sure to eval on 0th iteration
# but also after the very last iteration. so we loop for step <= num_iterations
# instead of just < num_iterations (one extra due to <=), only to do
# the validation/sampling one last time, and then we break right here as we're done.
if last_step:
break
# --------------- TRAINING SECTION BEGIN -----------------
model.train()
for i in range(1, train_accumulation_steps+1):
# forward pass
with ctx:
_, loss = model(x, y, return_logits=False)
train_loss = loss.detach()
# advance the dataset for the next batch
x, y = train_loader.next_batch()
# backward pass
if i < train_accumulation_steps:
with model.no_sync(): # there's no need to sync gradients every accumulation step
loss.backward()
else:
loss.backward() # just sync on the last step
for p in model.parameters():
p.grad /= train_accumulation_steps
# step the optimizers and schedulers
for opt, sched in zip(optimizers, schedulers):
opt.step()
sched.step()
# null the gradients
model.zero_grad(set_to_none=True)
# --------------- TRAINING SECTION END -------------------
# everything that follows now is just diagnostics, prints, logging, etc.
#dist.all_reduce(train_loss, op=dist.ReduceOp.AVG) # all-reducing the training loss would be more correct in terms of logging, but slower
if master_process:
approx_time = training_time_ms + 1000 * (time.time() - t0)
print(f"step:{step+1}/{args.num_iterations} train_loss:{train_loss.item():.4f} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")
with open(logfile, "a") as f:
f.write(f"step:{step+1}/{args.num_iterations} train_loss:{train_loss.item():.4f} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms\n")
if master_process:
print(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")
# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()
====================================================================================================
Running pytorch 2.4.1+cu121 compiled for CUDA 12.1
nvidia-smi:
Thu Oct 10 23:58:24 2024
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.129.03 Driver Version: 535.129.03 CUDA Version: 12.2 |
|-----------------------------------------+----------------------+----------------------+
| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|=========================================+======================+======================|
| 0 NVIDIA H100 80GB HBM3 On | 00000000:61:00.0 Off | 0 |
| N/A 33C P0 111W / 700W | 5789MiB / 81559MiB | 5% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
| 1 NVIDIA H100 80GB HBM3 On | 00000000:62:00.0 Off | 0 |
| N/A 30C P0 112W / 700W | 5837MiB / 81559MiB | 1% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
| 2 NVIDIA H100 80GB HBM3 On | 00000000:63:00.0 Off | 0 |
| N/A 29C P0 116W / 700W | 5837MiB / 81559MiB | 6% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
| 3 NVIDIA H100 80GB HBM3 On | 00000000:64:00.0 Off | 0 |
| N/A 32C P0 119W / 700W | 5837MiB / 81559MiB | 2% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
| 4 NVIDIA H100 80GB HBM3 On | 00000000:6A:00.0 Off | 0 |
| N/A 35C P0 115W / 700W | 5837MiB / 81559MiB | 4% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
| 5 NVIDIA H100 80GB HBM3 On | 00000000:6B:00.0 Off | 0 |
| N/A 31C P0 115W / 700W | 5837MiB / 81559MiB | 3% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
| 6 NVIDIA H100 80GB HBM3 On | 00000000:6C:00.0 Off | 0 |
| N/A 33C P0 117W / 700W | 5837MiB / 81559MiB | 3% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
| 7 NVIDIA H100 80GB HBM3 On | 00000000:6D:00.0 Off | 0 |
| N/A 29C P0 113W / 700W | 5597MiB / 81559MiB | 5% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
+---------------------------------------------------------------------------------------+
| Processes: |
| GPU GI CI PID Type Process name GPU Memory |
| ID ID Usage |
|=======================================================================================|
| 0 N/A N/A 6718 C /usr/bin/python3 5776MiB |
| 1 N/A N/A 6719 C /usr/bin/python3 5824MiB |
| 2 N/A N/A 6720 C /usr/bin/python3 5824MiB |
| 3 N/A N/A 6721 C /usr/bin/python3 5824MiB |
| 4 N/A N/A 6722 C /usr/bin/python3 5824MiB |
| 5 N/A N/A 6723 C /usr/bin/python3 5824MiB |
| 6 N/A N/A 6724 C /usr/bin/python3 5824MiB |
| 7 N/A N/A 6725 C /usr/bin/python3 5584MiB |
+---------------------------------------------------------------------------------------+
====================================================================================================
step:0/6200 val_loss:10.9264 train_time:285ms step_avg:nanms
step:1/6200 train_loss:10.9184 train_time:74701ms step_avg:nanms
step:2/6200 train_loss:8.6834 train_time:75965ms step_avg:nanms
step:3/6200 train_loss:7.7596 train_time:76178ms step_avg:nanms
step:4/6200 train_loss:7.5281 train_time:76393ms step_avg:nanms
step:5/6200 train_loss:7.2838 train_time:76607ms step_avg:nanms
step:6/6200 train_loss:7.4058 train_time:76822ms step_avg:nanms
step:7/6200 train_loss:7.1525 train_time:77034ms step_avg:nanms
step:8/6200 train_loss:7.2096 train_time:77248ms step_avg:nanms
step:9/6200 train_loss:6.9016 train_time:77462ms step_avg:nanms
step:10/6200 train_loss:6.7231 train_time:77676ms step_avg:nanms
step:11/6200 train_loss:6.6933 train_time:214ms step_avg:nanms
step:12/6200 train_loss:6.6519 train_time:428ms step_avg:nanms
step:13/6200 train_loss:6.5125 train_time:641ms step_avg:213.66ms
step:14/6200 train_loss:6.4803 train_time:854ms step_avg:213.44ms
step:15/6200 train_loss:6.4443 train_time:1067ms step_avg:213.35ms
step:16/6200 train_loss:6.3833 train_time:1280ms step_avg:213.27ms
step:17/6200 train_loss:6.3940 train_time:1494ms step_avg:213.38ms
step:18/6200 train_loss:6.4317 train_time:1706ms step_avg:213.31ms
step:19/6200 train_loss:6.2612 train_time:1920ms step_avg:213.34ms
step:20/6200 train_loss:6.2872 train_time:2134ms step_avg:213.39ms
step:21/6200 train_loss:5.9681 train_time:2348ms step_avg:213.42ms
step:22/6200 train_loss:6.3227 train_time:2559ms step_avg:213.28ms
step:23/6200 train_loss:6.5254 train_time:2771ms step_avg:213.18ms
step:24/6200 train_loss:6.2246 train_time:2984ms step_avg:213.17ms
step:25/6200 train_loss:6.3684 train_time:3197ms step_avg:213.12ms
step:26/6200 train_loss:6.0852 train_time:3417ms step_avg:213.54ms
step:27/6200 train_loss:5.9964 train_time:3631ms step_avg:213.58ms
step:28/6200 train_loss:6.1392 train_time:3843ms step_avg:213.52ms
step:29/6200 train_loss:5.8375 train_time:4055ms step_avg:213.43ms
step:30/6200 train_loss:6.1211 train_time:4268ms step_avg:213.40ms
step:31/6200 train_loss:5.9554 train_time:4481ms step_avg:213.38ms
step:32/6200 train_loss:5.9318 train_time:4695ms step_avg:213.40ms
step:33/6200 train_loss:5.7680 train_time:4909ms step_avg:213.43ms
step:34/6200 train_loss:6.0188 train_time:5122ms step_avg:213.41ms
step:35/6200 train_loss:5.9832 train_time:5336ms step_avg:213.43ms
step:36/6200 train_loss:6.1210 train_time:5548ms step_avg:213.40ms
step:37/6200 train_loss:6.0830 train_time:5760ms step_avg:213.35ms
step:38/6200 train_loss:5.9960 train_time:5974ms step_avg:213.35ms
step:39/6200 train_loss:5.8868 train_time:6187ms step_avg:213.35ms
step:40/6200 train_loss:5.9095 train_time:6399ms step_avg:213.31ms
step:41/6200 train_loss:5.8334 train_time:6614ms step_avg:213.35ms
step:42/6200 train_loss:5.8682 train_time:6827ms step_avg:213.34ms
step:43/6200 train_loss:5.7484 train_time:7039ms step_avg:213.30ms
step:44/6200 train_loss:5.8581 train_time:7252ms step_avg:213.29ms
step:45/6200 train_loss:5.8271 train_time:7464ms step_avg:213.26ms
step:46/6200 train_loss:5.9973 train_time:7677ms step_avg:213.26ms
step:47/6200 train_loss:5.7899 train_time:7891ms step_avg:213.26ms
step:48/6200 train_loss:5.6798 train_time:8103ms step_avg:213.24ms
step:49/6200 train_loss:5.8951 train_time:8317ms step_avg:213.27ms
step:50/6200 train_loss:5.7791 train_time:8532ms step_avg:213.31ms
step:51/6200 train_loss:5.9207 train_time:8746ms step_avg:213.31ms
step:52/6200 train_loss:5.8023 train_time:8957ms step_avg:213.27ms
step:53/6200 train_loss:5.6707 train_time:9170ms step_avg:213.27ms
step:54/6200 train_loss:5.7927 train_time:9383ms step_avg:213.24ms
step:55/6200 train_loss:5.7075 train_time:9597ms step_avg:213.26ms
step:56/6200 train_loss:6.0326 train_time:9810ms step_avg:213.26ms
step:57/6200 train_loss:5.7168 train_time:10024ms step_avg:213.27ms
step:58/6200 train_loss:5.5910 train_time:10237ms step_avg:213.27ms
step:59/6200 train_loss:5.7355 train_time:10451ms step_avg:213.28ms
step:60/6200 train_loss:5.7158 train_time:10665ms step_avg:213.30ms
step:61/6200 train_loss:5.7815 train_time:10878ms step_avg:213.29ms
step:62/6200 train_loss:5.6071 train_time:11091ms step_avg:213.29ms
step:63/6200 train_loss:5.7061 train_time:11305ms step_avg:213.30ms
step:64/6200 train_loss:5.6819 train_time:11519ms step_avg:213.32ms
step:65/6200 train_loss:5.2590 train_time:11735ms step_avg:213.36ms
step:66/6200 train_loss:5.5274 train_time:11948ms step_avg:213.35ms
step:67/6200 train_loss:5.6988 train_time:12160ms step_avg:213.34ms
step:68/6200 train_loss:5.5595 train_time:12374ms step_avg:213.34ms
step:69/6200 train_loss:5.7931 train_time:12587ms step_avg:213.34ms
step:70/6200 train_loss:5.4703 train_time:12799ms step_avg:213.32ms
step:71/6200 train_loss:5.4531 train_time:13014ms step_avg:213.34ms
step:72/6200 train_loss:5.7001 train_time:13228ms step_avg:213.36ms
step:73/6200 train_loss:5.6453 train_time:13441ms step_avg:213.35ms
step:74/6200 train_loss:5.5233 train_time:13654ms step_avg:213.34ms
step:75/6200 train_loss:5.6561 train_time:13867ms step_avg:213.34ms
step:76/6200 train_loss:5.5948 train_time:14080ms step_avg:213.33ms
step:77/6200 train_loss:5.5993 train_time:14294ms step_avg:213.34ms
step:78/6200 train_loss:5.6833 train_time:14508ms step_avg:213.36ms
step:79/6200 train_loss:5.7076 train_time:14721ms step_avg:213.35ms
step:80/6200 train_loss:5.5476 train_time:14935ms step_avg:213.36ms
step:81/6200 train_loss:5.6539 train_time:15149ms step_avg:213.37ms
step:82/6200 train_loss:5.4393 train_time:15362ms step_avg:213.36ms
step:83/6200 train_loss:5.6025 train_time:15576ms step_avg:213.36ms
step:84/6200 train_loss:5.5677 train_time:15790ms step_avg:213.37ms
step:85/6200 train_loss:5.5537 train_time:16003ms step_avg:213.38ms
step:86/6200 train_loss:5.4043 train_time:16217ms step_avg:213.39ms
step:87/6200 train_loss:5.6360 train_time:16432ms step_avg:213.41ms
step:88/6200 train_loss:5.5470 train_time:16646ms step_avg:213.41ms
step:89/6200 train_loss:5.5636 train_time:16859ms step_avg:213.41ms
step:90/6200 train_loss:5.5475 train_time:17072ms step_avg:213.40ms
step:91/6200 train_loss:5.4898 train_time:17286ms step_avg:213.41ms
step:92/6200 train_loss:5.4718 train_time:17500ms step_avg:213.41ms
step:93/6200 train_loss:5.5898 train_time:17714ms step_avg:213.42ms
step:94/6200 train_loss:5.4221 train_time:17930ms step_avg:213.45ms
step:95/6200 train_loss:5.4295 train_time:18144ms step_avg:213.46ms
step:96/6200 train_loss:5.4585 train_time:18356ms step_avg:213.45ms
step:97/6200 train_loss:5.3823 train_time:18570ms step_avg:213.45ms
step:98/6200 train_loss:5.4619 train_time:18784ms step_avg:213.45ms
step:99/6200 train_loss:5.3740 train_time:18997ms step_avg:213.45ms
step:100/6200 train_loss:5.4941 train_time:19212ms step_avg:213.47ms
step:101/6200 train_loss:5.4627 train_time:19427ms step_avg:213.48ms
step:102/6200 train_loss:5.3584 train_time:19639ms step_avg:213.47ms
step:103/6200 train_loss:5.4707 train_time:19853ms step_avg:213.47ms
step:104/6200 train_loss:5.3986 train_time:20066ms step_avg:213.47ms
step:105/6200 train_loss:5.2556 train_time:20280ms step_avg:213.47ms
step:106/6200 train_loss:5.3758 train_time:20495ms step_avg:213.49ms
step:107/6200 train_loss:5.5572 train_time:20709ms step_avg:213.50ms
step:108/6200 train_loss:5.3541 train_time:20923ms step_avg:213.50ms
step:109/6200 train_loss:5.1235 train_time:21137ms step_avg:213.50ms
step:110/6200 train_loss:5.3207 train_time:21351ms step_avg:213.51ms
step:111/6200 train_loss:5.2965 train_time:21564ms step_avg:213.50ms
step:112/6200 train_loss:5.2638 train_time:21778ms step_avg:213.51ms
step:113/6200 train_loss:5.3656 train_time:21992ms step_avg:213.52ms
step:114/6200 train_loss:5.2965 train_time:22207ms step_avg:213.53ms
step:115/6200 train_loss:5.1582 train_time:22421ms step_avg:213.54ms
step:116/6200 train_loss:5.3360 train_time:22636ms step_avg:213.55ms
step:117/6200 train_loss:5.2026 train_time:22850ms step_avg:213.55ms
step:118/6200 train_loss:5.1728 train_time:23063ms step_avg:213.55ms
step:119/6200 train_loss:5.2965 train_time:23278ms step_avg:213.56ms
step:120/6200 train_loss:5.2795 train_time:23494ms step_avg:213.58ms
step:121/6200 train_loss:5.2036 train_time:23708ms step_avg:213.59ms
step:122/6200 train_loss:5.1102 train_time:23921ms step_avg:213.58ms
step:123/6200 train_loss:5.2210 train_time:24137ms step_avg:213.60ms
step:124/6200 train_loss:5.0597 train_time:24349ms step_avg:213.59ms
step:125/6200 train_loss:5.3717 train_time:24562ms step_avg:213.58ms
step:125/6200 val_loss:5.2017 train_time:24564ms step_avg:213.60ms
step:126/6200 train_loss:5.2399 train_time:24781ms step_avg:213.63ms
step:127/6200 train_loss:5.1921 train_time:24997ms step_avg:213.65ms
step:128/6200 train_loss:5.2629 train_time:25211ms step_avg:213.65ms
step:129/6200 train_loss:5.1224 train_time:25425ms step_avg:213.65ms
step:130/6200 train_loss:5.4064 train_time:25638ms step_avg:213.65ms
step:131/6200 train_loss:5.1800 train_time:25852ms step_avg:213.65ms
step:132/6200 train_loss:5.1772 train_time:26065ms step_avg:213.65ms
step:133/6200 train_loss:5.1321 train_time:26280ms step_avg:213.66ms
step:134/6200 train_loss:5.1733 train_time:26495ms step_avg:213.67ms
step:135/6200 train_loss:5.0687 train_time:26709ms step_avg:213.68ms
step:136/6200 train_loss:5.1770 train_time:26924ms step_avg:213.68ms
step:137/6200 train_loss:4.9614 train_time:27138ms step_avg:213.69ms
step:138/6200 train_loss:5.1265 train_time:27352ms step_avg:213.69ms
step:139/6200 train_loss:5.0653 train_time:27568ms step_avg:213.70ms
step:140/6200 train_loss:5.1074 train_time:27782ms step_avg:213.71ms
step:141/6200 train_loss:5.1656 train_time:27998ms step_avg:213.72ms
step:142/6200 train_loss:5.0510 train_time:28211ms step_avg:213.72ms
step:143/6200 train_loss:5.1072 train_time:28425ms step_avg:213.72ms
step:144/6200 train_loss:4.9479 train_time:28639ms step_avg:213.72ms
step:145/6200 train_loss:5.1043 train_time:28853ms step_avg:213.73ms
step:146/6200 train_loss:5.0387 train_time:29068ms step_avg:213.74ms
step:147/6200 train_loss:4.9190 train_time:29283ms step_avg:213.75ms
step:148/6200 train_loss:5.0692 train_time:29499ms step_avg:213.76ms
step:149/6200 train_loss:5.0601 train_time:29714ms step_avg:213.77ms
step:150/6200 train_loss:5.0808 train_time:29927ms step_avg:213.76ms
step:151/6200 train_loss:5.1169 train_time:30140ms step_avg:213.76ms
step:152/6200 train_loss:5.0243 train_time:30354ms step_avg:213.76ms
step:153/6200 train_loss:5.0093 train_time:30568ms step_avg:213.76ms
step:154/6200 train_loss:5.1044 train_time:30782ms step_avg:213.77ms
step:155/6200 train_loss:5.0418 train_time:30998ms step_avg:213.78ms
step:156/6200 train_loss:5.0039 train_time:31211ms step_avg:213.77ms
step:157/6200 train_loss:5.0335 train_time:31425ms step_avg:213.77ms
step:158/6200 train_loss:5.1590 train_time:31638ms step_avg:213.77ms
step:159/6200 train_loss:4.9323 train_time:31851ms step_avg:213.77ms
step:160/6200 train_loss:4.9939 train_time:32065ms step_avg:213.77ms
step:161/6200 train_loss:4.8373 train_time:32281ms step_avg:213.78ms
step:162/6200 train_loss:5.0129 train_time:32495ms step_avg:213.78ms
step:163/6200 train_loss:5.0512 train_time:32708ms step_avg:213.78ms
step:164/6200 train_loss:5.0302 train_time:32921ms step_avg:213.77ms
step:165/6200 train_loss:4.8465 train_time:33135ms step_avg:213.77ms
step:166/6200 train_loss:4.9743 train_time:33349ms step_avg:213.78ms
step:167/6200 train_loss:5.1055 train_time:33564ms step_avg:213.78ms
step:168/6200 train_loss:4.8925 train_time:33779ms step_avg:213.79ms
step:169/6200 train_loss:4.9942 train_time:33996ms step_avg:213.81ms
step:170/6200 train_loss:4.8351 train_time:34209ms step_avg:213.81ms
step:171/6200 train_loss:4.7438 train_time:34422ms step_avg:213.80ms
step:172/6200 train_loss:4.8904 train_time:34637ms step_avg:213.81ms
step:173/6200 train_loss:4.8800 train_time:34850ms step_avg:213.81ms
step:174/6200 train_loss:4.9316 train_time:35065ms step_avg:213.81ms
step:175/6200 train_loss:5.0703 train_time:35280ms step_avg:213.82ms
step:176/6200 train_loss:4.9360 train_time:35493ms step_avg:213.82ms
step:177/6200 train_loss:4.7875 train_time:35707ms step_avg:213.82ms
step:178/6200 train_loss:4.7673 train_time:35921ms step_avg:213.81ms
step:179/6200 train_loss:4.8106 train_time:36134ms step_avg:213.81ms
step:180/6200 train_loss:4.8333 train_time:36349ms step_avg:213.81ms
step:181/6200 train_loss:4.8279 train_time:36564ms step_avg:213.82ms
step:182/6200 train_loss:4.9477 train_time:36779ms step_avg:213.83ms
step:183/6200 train_loss:4.8408 train_time:36993ms step_avg:213.83ms
step:184/6200 train_loss:4.7580 train_time:37207ms step_avg:213.84ms
step:185/6200 train_loss:4.7972 train_time:37421ms step_avg:213.83ms
step:186/6200 train_loss:4.9072 train_time:37634ms step_avg:213.83ms
step:187/6200 train_loss:4.8109 train_time:37849ms step_avg:213.83ms
step:188/6200 train_loss:5.0394 train_time:38065ms step_avg:213.85ms
step:189/6200 train_loss:4.8372 train_time:38526ms step_avg:215.23ms
step:190/6200 train_loss:4.7508 train_time:38989ms step_avg:216.61ms
step:191/6200 train_loss:4.9076 train_time:39205ms step_avg:216.60ms
step:192/6200 train_loss:4.7564 train_time:39419ms step_avg:216.59ms
step:193/6200 train_loss:4.6693 train_time:39634ms step_avg:216.58ms
step:194/6200 train_loss:4.8827 train_time:39849ms step_avg:216.57ms
step:195/6200 train_loss:4.8262 train_time:40063ms step_avg:216.56ms
step:196/6200 train_loss:5.0026 train_time:40278ms step_avg:216.55ms
step:197/6200 train_loss:4.8898 train_time:40493ms step_avg:216.54ms
step:198/6200 train_loss:4.7291 train_time:40707ms step_avg:216.53ms
step:199/6200 train_loss:4.7881 train_time:40921ms step_avg:216.51ms
step:200/6200 train_loss:4.6653 train_time:41135ms step_avg:216.50ms
step:201/6200 train_loss:4.7457 train_time:41349ms step_avg:216.49ms
step:202/6200 train_loss:4.6478 train_time:41563ms step_avg:216.48ms
step:203/6200 train_loss:4.9032 train_time:41779ms step_avg:216.47ms
step:204/6200 train_loss:4.7975 train_time:41993ms step_avg:216.46ms
step:205/6200 train_loss:4.7753 train_time:42207ms step_avg:216.45ms
step:206/6200 train_loss:4.9187 train_time:42421ms step_avg:216.43ms
step:207/6200 train_loss:4.5771 train_time:42634ms step_avg:216.42ms
step:208/6200 train_loss:4.7399 train_time:42849ms step_avg:216.41ms
step:209/6200 train_loss:4.6904 train_time:43064ms step_avg:216.40ms
step:210/6200 train_loss:4.8590 train_time:43279ms step_avg:216.39ms
step:211/6200 train_loss:4.7850 train_time:43493ms step_avg:216.38ms
step:212/6200 train_loss:4.6588 train_time:43707ms step_avg:216.37ms
step:213/6200 train_loss:4.8168 train_time:43921ms step_avg:216.36ms
step:214/6200 train_loss:4.6370 train_time:44134ms step_avg:216.34ms
step:215/6200 train_loss:4.7172 train_time:44349ms step_avg:216.33ms
step:216/6200 train_loss:4.5746 train_time:44563ms step_avg:216.33ms
step:217/6200 train_loss:4.7150 train_time:44779ms step_avg:216.32ms
step:218/6200 train_loss:4.6749 train_time:44994ms step_avg:216.32ms
step:219/6200 train_loss:4.6666 train_time:45208ms step_avg:216.30ms
step:220/6200 train_loss:4.6734 train_time:45420ms step_avg:216.29ms
step:221/6200 train_loss:4.7200 train_time:45634ms step_avg:216.27ms
step:222/6200 train_loss:4.7403 train_time:45849ms step_avg:216.27ms
step:223/6200 train_loss:4.6556 train_time:46063ms step_avg:216.26ms
step:224/6200 train_loss:4.6782 train_time:46278ms step_avg:216.25ms
step:225/6200 train_loss:4.8179 train_time:46492ms step_avg:216.24ms
step:226/6200 train_loss:4.5556 train_time:46706ms step_avg:216.23ms
step:227/6200 train_loss:4.5680 train_time:46920ms step_avg:216.22ms
step:228/6200 train_loss:4.5619 train_time:47134ms step_avg:216.21ms
step:229/6200 train_loss:4.7298 train_time:47348ms step_avg:216.20ms
step:230/6200 train_loss:4.5533 train_time:47563ms step_avg:216.20ms
step:231/6200 train_loss:4.6905 train_time:47779ms step_avg:216.19ms
step:232/6200 train_loss:4.5586 train_time:47992ms step_avg:216.18ms
step:233/6200 train_loss:4.5362 train_time:48206ms step_avg:216.17ms
step:234/6200 train_loss:4.7408 train_time:48421ms step_avg:216.16ms
step:235/6200 train_loss:4.5758 train_time:48634ms step_avg:216.15ms
step:236/6200 train_loss:4.5213 train_time:48848ms step_avg:216.14ms
step:237/6200 train_loss:4.7476 train_time:49063ms step_avg:216.14ms
step:238/6200 train_loss:4.6531 train_time:49277ms step_avg:216.13ms
step:239/6200 train_loss:4.5491 train_time:49491ms step_avg:216.12ms
step:240/6200 train_loss:4.7010 train_time:49705ms step_avg:216.11ms
step:241/6200 train_loss:4.6761 train_time:49919ms step_avg:216.10ms
step:242/6200 train_loss:4.5845 train_time:50133ms step_avg:216.09ms
step:243/6200 train_loss:4.7437 train_time:50347ms step_avg:216.08ms
step:244/6200 train_loss:4.5735 train_time:50562ms step_avg:216.08ms
step:245/6200 train_loss:4.6053 train_time:50777ms step_avg:216.07ms
step:246/6200 train_loss:4.6616 train_time:50991ms step_avg:216.06ms
step:247/6200 train_loss:4.6211 train_time:51205ms step_avg:216.06ms
step:248/6200 train_loss:4.5680 train_time:51422ms step_avg:216.06ms
step:249/6200 train_loss:4.7262 train_time:51634ms step_avg:216.04ms
step:250/6200 train_loss:4.4653 train_time:51849ms step_avg:216.04ms
step:250/6200 val_loss:4.5819 train_time:51851ms step_avg:216.04ms
step:251/6200 train_loss:4.5217 train_time:52066ms step_avg:216.04ms
step:252/6200 train_loss:4.6475 train_time:52279ms step_avg:216.03ms
step:253/6200 train_loss:4.6471 train_time:52493ms step_avg:216.02ms
step:254/6200 train_loss:4.5112 train_time:52707ms step_avg:216.01ms
step:255/6200 train_loss:4.5158 train_time:52923ms step_avg:216.01ms
step:256/6200 train_loss:4.6784 train_time:53139ms step_avg:216.01ms
step:257/6200 train_loss:4.6019 train_time:53353ms step_avg:216.01ms
step:258/6200 train_loss:4.5689 train_time:53567ms step_avg:216.00ms
step:259/6200 train_loss:4.5131 train_time:53781ms step_avg:215.99ms
step:260/6200 train_loss:4.5425 train_time:53997ms step_avg:215.99ms
step:261/6200 train_loss:4.6066 train_time:54211ms step_avg:215.98ms
step:262/6200 train_loss:4.5973 train_time:54424ms step_avg:215.97ms
step:263/6200 train_loss:4.5266 train_time:54640ms step_avg:215.97ms
step:264/6200 train_loss:4.4474 train_time:54855ms step_avg:215.97ms
step:265/6200 train_loss:4.5160 train_time:55069ms step_avg:215.96ms
step:266/6200 train_loss:4.3566 train_time:55283ms step_avg:215.95ms
step:267/6200 train_loss:4.4277 train_time:55497ms step_avg:215.94ms
step:268/6200 train_loss:4.4618 train_time:55713ms step_avg:215.94ms
step:269/6200 train_loss:4.4411 train_time:55928ms step_avg:215.94ms
step:270/6200 train_loss:4.3782 train_time:56143ms step_avg:215.93ms
step:271/6200 train_loss:4.6304 train_time:56358ms step_avg:215.93ms
step:272/6200 train_loss:4.5307 train_time:56571ms step_avg:215.92ms
step:273/6200 train_loss:4.4100 train_time:56785ms step_avg:215.91ms
step:274/6200 train_loss:4.4493 train_time:57000ms step_avg:215.91ms
step:275/6200 train_loss:4.5645 train_time:57215ms step_avg:215.91ms
step:276/6200 train_loss:4.5687 train_time:57429ms step_avg:215.90ms
step:277/6200 train_loss:4.7986 train_time:57643ms step_avg:215.89ms
step:278/6200 train_loss:4.5276 train_time:57861ms step_avg:215.90ms
step:279/6200 train_loss:4.6538 train_time:58075ms step_avg:215.89ms
step:280/6200 train_loss:4.5040 train_time:58288ms step_avg:215.88ms
step:281/6200 train_loss:4.5974 train_time:58503ms step_avg:215.88ms
step:282/6200 train_loss:4.4633 train_time:58718ms step_avg:215.87ms
step:283/6200 train_loss:4.5411 train_time:58932ms step_avg:215.87ms
step:284/6200 train_loss:4.3905 train_time:59147ms step_avg:215.86ms
step:285/6200 train_loss:4.5661 train_time:59361ms step_avg:215.86ms
step:286/6200 train_loss:4.5567 train_time:59576ms step_avg:215.86ms
step:287/6200 train_loss:4.5842 train_time:59789ms step_avg:215.84ms
step:288/6200 train_loss:4.4474 train_time:60003ms step_avg:215.84ms
step:289/6200 train_loss:4.5033 train_time:60218ms step_avg:215.84ms
step:290/6200 train_loss:4.3760 train_time:60436ms step_avg:215.84ms
step:291/6200 train_loss:4.3604 train_time:60650ms step_avg:215.84ms
step:292/6200 train_loss:4.4704 train_time:60865ms step_avg:215.83ms
step:293/6200 train_loss:4.3692 train_time:61078ms step_avg:215.82ms
step:294/6200 train_loss:4.4265 train_time:61292ms step_avg:215.82ms
step:295/6200 train_loss:4.4373 train_time:61506ms step_avg:215.81ms
step:296/6200 train_loss:4.3125 train_time:61720ms step_avg:215.81ms
step:297/6200 train_loss:4.2993 train_time:61935ms step_avg:215.80ms
step:298/6200 train_loss:4.3241 train_time:62150ms step_avg:215.80ms
step:299/6200 train_loss:4.4313 train_time:62364ms step_avg:215.79ms
step:300/6200 train_loss:4.3087 train_time:62578ms step_avg:215.79ms
step:301/6200 train_loss:4.4787 train_time:62791ms step_avg:215.78ms
step:302/6200 train_loss:4.4615 train_time:63007ms step_avg:215.78ms
step:303/6200 train_loss:4.3868 train_time:63222ms step_avg:215.77ms
step:304/6200 train_loss:4.4663 train_time:63437ms step_avg:215.77ms
step:305/6200 train_loss:4.4402 train_time:63650ms step_avg:215.76ms
step:306/6200 train_loss:4.9214 train_time:63864ms step_avg:215.76ms
step:307/6200 train_loss:4.3958 train_time:64078ms step_avg:215.75ms
step:308/6200 train_loss:4.2926 train_time:64292ms step_avg:215.74ms
step:309/6200 train_loss:4.4803 train_time:64506ms step_avg:215.74ms
step:310/6200 train_loss:4.2876 train_time:64720ms step_avg:215.73ms
step:311/6200 train_loss:4.5223 train_time:64935ms step_avg:215.73ms
step:312/6200 train_loss:4.4093 train_time:65150ms step_avg:215.73ms
step:313/6200 train_loss:4.3311 train_time:65364ms step_avg:215.72ms
step:314/6200 train_loss:4.4214 train_time:65579ms step_avg:215.72ms
step:315/6200 train_loss:4.5724 train_time:65792ms step_avg:215.71ms
step:316/6200 train_loss:4.4249 train_time:66006ms step_avg:215.71ms
step:317/6200 train_loss:4.2896 train_time:66220ms step_avg:215.70ms
step:318/6200 train_loss:4.3191 train_time:66435ms step_avg:215.70ms
step:319/6200 train_loss:4.3480 train_time:66650ms step_avg:215.70ms
step:320/6200 train_loss:4.3070 train_time:66865ms step_avg:215.69ms
step:321/6200 train_loss:4.3968 train_time:67079ms step_avg:215.69ms
step:322/6200 train_loss:4.3938 train_time:67292ms step_avg:215.68ms
step:323/6200 train_loss:4.3494 train_time:67506ms step_avg:215.67ms
step:324/6200 train_loss:4.4331 train_time:67720ms step_avg:215.67ms
step:325/6200 train_loss:4.4043 train_time:67941ms step_avg:215.68ms
step:326/6200 train_loss:4.4620 train_time:68156ms step_avg:215.68ms
step:327/6200 train_loss:4.3255 train_time:68371ms step_avg:215.68ms
step:328/6200 train_loss:4.8162 train_time:68584ms step_avg:215.67ms
step:329/6200 train_loss:4.4895 train_time:68798ms step_avg:215.67ms
step:330/6200 train_loss:4.2489 train_time:69011ms step_avg:215.66ms
step:331/6200 train_loss:4.2061 train_time:69225ms step_avg:215.65ms
step:332/6200 train_loss:4.3983 train_time:69440ms step_avg:215.65ms
step:333/6200 train_loss:4.3157 train_time:69655ms step_avg:215.65ms
step:334/6200 train_loss:4.2998 train_time:69868ms step_avg:215.64ms
step:335/6200 train_loss:4.2619 train_time:70082ms step_avg:215.64ms
step:336/6200 train_loss:4.4407 train_time:70298ms step_avg:215.64ms
step:337/6200 train_loss:4.3790 train_time:70512ms step_avg:215.63ms
step:338/6200 train_loss:4.9143 train_time:70727ms step_avg:215.63ms
step:339/6200 train_loss:4.3475 train_time:70941ms step_avg:215.63ms
step:340/6200 train_loss:4.3131 train_time:71157ms step_avg:215.63ms
step:341/6200 train_loss:4.3150 train_time:71371ms step_avg:215.62ms
step:342/6200 train_loss:4.2503 train_time:71585ms step_avg:215.62ms
step:343/6200 train_loss:4.2176 train_time:71799ms step_avg:215.61ms
step:344/6200 train_loss:4.2804 train_time:72013ms step_avg:215.61ms
step:345/6200 train_loss:4.3851 train_time:72227ms step_avg:215.60ms
step:346/6200 train_loss:4.2452 train_time:72442ms step_avg:215.60ms
step:347/6200 train_loss:4.1847 train_time:72657ms step_avg:215.60ms
step:348/6200 train_loss:4.2397 train_time:72870ms step_avg:215.59ms
step:349/6200 train_loss:4.2508 train_time:73088ms step_avg:215.60ms
step:350/6200 train_loss:4.1974 train_time:73302ms step_avg:215.60ms
step:351/6200 train_loss:3.9017 train_time:73519ms step_avg:215.60ms
step:352/6200 train_loss:4.1872 train_time:73734ms step_avg:215.60ms
step:353/6200 train_loss:4.5296 train_time:73948ms step_avg:215.59ms
step:354/6200 train_loss:4.0563 train_time:74161ms step_avg:215.59ms
step:355/6200 train_loss:4.3017 train_time:74377ms step_avg:215.58ms
step:356/6200 train_loss:4.1954 train_time:74590ms step_avg:215.58ms
step:357/6200 train_loss:4.2777 train_time:74804ms step_avg:215.57ms
step:358/6200 train_loss:4.2728 train_time:75020ms step_avg:215.58ms
step:359/6200 train_loss:4.2246 train_time:75235ms step_avg:215.57ms
step:360/6200 train_loss:4.4278 train_time:75449ms step_avg:215.57ms
step:361/6200 train_loss:3.8887 train_time:75664ms step_avg:215.57ms
step:362/6200 train_loss:4.4090 train_time:75877ms step_avg:215.56ms
step:363/6200 train_loss:4.3100 train_time:76091ms step_avg:215.56ms
step:364/6200 train_loss:4.2067 train_time:76307ms step_avg:215.56ms
step:365/6200 train_loss:4.1359 train_time:76521ms step_avg:215.55ms
step:366/6200 train_loss:4.2958 train_time:76736ms step_avg:215.55ms
step:367/6200 train_loss:4.2354 train_time:76950ms step_avg:215.55ms
step:368/6200 train_loss:4.2165 train_time:77164ms step_avg:215.54ms
step:369/6200 train_loss:4.2089 train_time:77378ms step_avg:215.54ms
step:370/6200 train_loss:4.1071 train_time:77592ms step_avg:215.53ms
step:371/6200 train_loss:4.2455 train_time:77806ms step_avg:215.53ms
step:372/6200 train_loss:4.1458 train_time:78021ms step_avg:215.53ms
step:373/6200 train_loss:4.0545 train_time:78235ms step_avg:215.52ms
step:374/6200 train_loss:4.2650 train_time:78449ms step_avg:215.52ms
step:375/6200 train_loss:4.1909 train_time:78664ms step_avg:215.52ms
step:375/6200 val_loss:4.1992 train_time:78666ms step_avg:215.52ms
step:376/6200 train_loss:4.1725 train_time:78882ms step_avg:215.52ms
step:377/6200 train_loss:4.2353 train_time:79097ms step_avg:215.52ms
step:378/6200 train_loss:4.1450 train_time:79561ms step_avg:216.20ms
step:379/6200 train_loss:4.2014 train_time:79775ms step_avg:216.19ms
step:380/6200 train_loss:4.2431 train_time:80238ms step_avg:216.86ms
step:381/6200 train_loss:4.2982 train_time:80452ms step_avg:216.85ms
step:382/6200 train_loss:4.2087 train_time:80665ms step_avg:216.84ms
step:383/6200 train_loss:4.1964 train_time:80879ms step_avg:216.83ms
step:384/6200 train_loss:4.1321 train_time:81094ms step_avg:216.83ms
step:385/6200 train_loss:4.2195 train_time:81309ms step_avg:216.82ms
step:386/6200 train_loss:4.1309 train_time:81523ms step_avg:216.82ms
step:387/6200 train_loss:4.2607 train_time:81737ms step_avg:216.81ms
step:388/6200 train_loss:4.4492 train_time:81951ms step_avg:216.80ms
step:389/6200 train_loss:4.1444 train_time:82165ms step_avg:216.80ms
step:390/6200 train_loss:4.1330 train_time:82379ms step_avg:216.79ms
step:391/6200 train_loss:4.2370 train_time:82594ms step_avg:216.78ms
step:392/6200 train_loss:4.1600 train_time:82809ms step_avg:216.78ms
step:393/6200 train_loss:4.2674 train_time:83023ms step_avg:216.77ms
step:394/6200 train_loss:4.0897 train_time:83236ms step_avg:216.76ms
step:395/6200 train_loss:4.2283 train_time:83451ms step_avg:216.76ms
step:396/6200 train_loss:3.9827 train_time:83665ms step_avg:216.75ms
step:397/6200 train_loss:4.1742 train_time:83878ms step_avg:216.74ms
step:398/6200 train_loss:4.2463 train_time:84093ms step_avg:216.73ms
step:399/6200 train_loss:4.2290 train_time:84307ms step_avg:216.73ms
step:400/6200 train_loss:4.1227 train_time:84521ms step_avg:216.72ms
step:401/6200 train_loss:4.2022 train_time:84735ms step_avg:216.71ms
step:402/6200 train_loss:4.2377 train_time:84949ms step_avg:216.71ms
step:403/6200 train_loss:4.1906 train_time:85163ms step_avg:216.70ms
step:404/6200 train_loss:4.2942 train_time:85378ms step_avg:216.70ms
step:405/6200 train_loss:4.0573 train_time:85593ms step_avg:216.69ms
step:406/6200 train_loss:4.1292 train_time:85808ms step_avg:216.69ms
step:407/6200 train_loss:4.4128 train_time:86021ms step_avg:216.68ms
step:408/6200 train_loss:4.1440 train_time:86235ms step_avg:216.67ms
step:409/6200 train_loss:4.1575 train_time:86449ms step_avg:216.66ms
step:410/6200 train_loss:4.2027 train_time:86662ms step_avg:216.66ms
step:411/6200 train_loss:4.0809 train_time:86877ms step_avg:216.65ms