forked from KellerJordan/modded-nanogpt
-
Notifications
You must be signed in to change notification settings - Fork 0
/
dabaaddd-237c-4ec9-939d-6608a9ed5e27.txt
5719 lines (5657 loc) · 383 KB
/
dabaaddd-237c-4ec9-939d-6608a9ed5e27.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
====================================================================================================
import os
import sys
with open(sys.argv[0]) as f:
code = f.read() # read the code of this file ASAP, for logging
import uuid
import glob
import time
from dataclasses import dataclass
import numpy as np
import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
# -----------------------------------------------------------------------------
# Muon optimizer
def zeropower_via_svd(G, steps=None):
U, S, V = G.svd()
return U @ V.T
@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
"""
Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
zero even beyond the point where the iteration no longer converges all the way to one everywhere
on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
where S' is diagonal with S_{ii}' \sim Uniform(0.5, 1.5), which turns out not to hurt model
performance at all relative to UV^T, where USV^T = G is the SVD.
"""
assert len(G.shape) == 2
a, b, c = (3.4445, -4.7750, 2.0315)
X = G.bfloat16()
X /= (X.norm() + eps) # ensure top singular value <= 1
if G.size(0) > G.size(1):
X = X.T
for _ in range(steps):
A = X @ X.T
B = A @ X
X = a * X + b * B + c * A @ B
if G.size(0) > G.size(1):
X = X.T
return X
zeropower_backends = dict(svd=zeropower_via_svd, newtonschulz5=zeropower_via_newtonschulz5)
class Muon(torch.optim.Optimizer):
"""
Muon - MomentUm Orthogonalized by Newton-schulz
Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
the advantage that it can be stably run in bfloat16 on the GPU.
Some warnings:
- This optimizer assumes that all parameters passed in are 2D.
- It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
parameters; those should all be optimized by a standard method (e.g., AdamW).
- To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
- We believe it is unlikely to work well for training with small batch size.
- We believe it may not work well for finetuning pretrained models, but we haven't tested this.
- We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).
Arguments:
lr: The learning rate used by the internal SGD.
momentum: The momentum used by the internal SGD.
nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
backend: The chosen backend for the orthogonalization step. (recommended: 'newtonschulz5')
backend_steps: The number of iteration steps to use in the backend, if it is iterative.
"""
def __init__(self, params, lr=3e-4, momentum=0.95, nesterov=True, backend='newtonschulz5', backend_steps=5):
defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, backend=backend, backend_steps=backend_steps)
super().__init__(params, defaults)
def step(self):
for group in self.param_groups:
lr = group['lr']
momentum = group['momentum']
zeropower_backend = zeropower_backends[group['backend']]
for p in group['params']:
g = p.grad
if g is None:
continue
state = self.state[p]
if 'momentum_buffer' not in state:
state['momentum_buffer'] = torch.zeros_like(g)
buf = state['momentum_buffer']
buf.mul_(momentum).add_(g)
if group['nesterov']:
g = g.add(buf, alpha=momentum)
if g.size(0) == 3 * g.size(1): # split grouped QKV parameters
g = torch.cat([zeropower_backend(g1, steps=group['backend_steps']) for g1 in g.split(g.size(1))])
scale = g.size(1)**0.5
else:
g = zeropower_backend(g, steps=group['backend_steps'])
scale = max(g.size(0), g.size(1))**0.5 # scale to have update.square().mean() == 1
p.data.add_(g, alpha=-lr * scale)
# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model
class Rotary(torch.nn.Module):
def __init__(self, dim, base=10000):
super().__init__()
self.inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
self.seq_len_cached = None
self.cos_cached = None
self.sin_cached = None
def forward(self, x):
seq_len = x.shape[1]
if seq_len != self.seq_len_cached:
self.seq_len_cached = seq_len
t = torch.arange(seq_len, device=x.device).type_as(self.inv_freq)
freqs = torch.outer(t, self.inv_freq).to(x.device)
self.cos_cached = freqs.cos().bfloat16()
self.sin_cached = freqs.sin().bfloat16()
return self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
def apply_rotary_emb(x, cos, sin):
assert x.ndim == 4 # multihead attention
d = x.shape[3]//2
x1 = x[..., :d]
x2 = x[..., d:]
y1 = x1 * cos + x2 * sin
y2 = x1 * (-sin) + x2 * cos
return torch.cat([y1, y2], 3).type_as(x)
class CausalSelfAttention(nn.Module):
def __init__(self, config):
super().__init__()
self.n_head = config.n_head
self.n_embd = config.n_embd
self.head_dim = self.n_embd // self.n_head
assert self.n_embd % self.n_head == 0
self.c_q = nn.Linear(self.n_embd, self.n_embd, bias=False)
self.c_k = nn.Linear(self.n_embd, self.n_embd, bias=False)
self.c_v = nn.Linear(self.n_embd, self.n_embd, bias=False)
# output projection
self.c_proj = nn.Linear(self.n_embd, self.n_embd, bias=False)
self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977
self.rotary = Rotary(self.head_dim)
def forward(self, x):
B, T, C = x.size() # batch size, sequence length, embedding dimensionality (n_embd)
q = self.c_q(x).view(B, T, self.n_head, self.head_dim)
k = self.c_k(x).view(B, T, self.n_head, self.head_dim)
v = self.c_v(x).view(B, T, self.n_head, self.head_dim)
cos, sin = self.rotary(q)
q, k = apply_rotary_emb(q, cos, sin), apply_rotary_emb(k, cos, sin)
q, k = F.rms_norm(q, (q.size(-1),)), F.rms_norm(k, (k.size(-1),)) # QK norm suggested by @Grad62304977
y = F.scaled_dot_product_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), is_causal=True)
y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
y = self.c_proj(y)
return y
class MLP(nn.Module):
def __init__(self, config):
super().__init__()
self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd, bias=False)
self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd, bias=False)
self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977
def forward(self, x):
x = self.c_fc(x)
x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
x = self.c_proj(x)
return x
class Block(nn.Module):
def __init__(self, config):
super().__init__()
self.attn = CausalSelfAttention(config)
self.mlp = MLP(config)
def forward(self, x):
x = x + self.attn(F.rms_norm(x, (x.size(-1),)))
x = x + self.mlp(F.rms_norm(x, (x.size(-1),)))
return x
# -----------------------------------------------------------------------------
# The main GPT-2 model
@dataclass
class GPTConfig:
vocab_size : int = 50304
n_layer : int = 12
n_head : int = 6 # head dim 128 suggested by @Grad62304977
n_embd : int = 768
class GPT(nn.Module):
def __init__(self, config):
super().__init__()
self.config = config
self.transformer = nn.ModuleDict(dict(
wte = nn.Embedding(config.vocab_size, config.n_embd),
h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
))
self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
self.transformer.wte.weight = self.lm_head.weight # https://paperswithcode.com/method/weight-tying
def forward(self, idx, targets=None, return_logits=True):
# forward the GPT model itself
x = self.transformer.wte(idx) # token embeddings of shape (b, t, n_embd)
for block in self.transformer.h:
x = block(x)
x = F.rms_norm(x, (x.size(-1),))
if targets is not None:
# if we are given some desired targets also calculate the loss
logits = self.lm_head(x)
logits = logits.float() # use tf32/fp32 for logits
loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1)
else:
# inference-time mini-optimization: only forward the lm_head on the very last position
logits = self.lm_head(x[:, [-1], :]) # note: using list [-1] to preserve the time dim
logits = logits.float() # use tf32/fp32 for logits
loss = None
# there are performance reasons why not returning logits is prudent, if not needed
if not return_logits:
logits = None
return logits, loss
# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader
def _peek_data_shard(filename):
# only reads the header, returns header data
with open(filename, "rb") as f:
# first read the header, which is 256 int32 integers (4 bytes each)
header = np.frombuffer(f.read(256*4), dtype=np.int32)
if header[0] != 20240520:
print("ERROR: magic number mismatch in the data .bin file!")
print("---> HINT: Are you passing in a correct file with --input_bin?")
print("---> HINT: Dataset encoding changed recently, re-run data prepro or refer again to README")
print("---> HINT: For example re-run: `python dev/data/tinyshakespeare.py`, then re-try")
exit(1)
assert header[1] == 1, "unsupported version"
ntok = header[2] # number of tokens (claimed)
return ntok # for now just return the number of tokens
def _load_data_shard(filename):
with open(filename, "rb") as f:
# first read the header, which is 256 int32 integers (4 bytes each)
header = np.frombuffer(f.read(256*4), dtype=np.int32)
assert header[0] == 20240520, "magic number mismatch in the data .bin file"
assert header[1] == 1, "unsupported version"
ntok = header[2] # number of tokens (claimed)
# the rest of it are tokens, stored as uint16
tokens = np.frombuffer(f.read(), dtype=np.uint16)
assert len(tokens) == ntok, "number of tokens read does not match header?"
return tokens
class DistributedDataLoader:
def __init__(self, filename_pattern, B, T, process_rank, num_processes):
self.process_rank = process_rank
self.num_processes = num_processes
self.B = B
self.T = T
# glob files that match the pattern
self.files = sorted(glob.glob(filename_pattern))
assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"
# load and validate all data shards, count number of tokens in total
ntok_total = 0
for fname in self.files:
shard_ntok = _peek_data_shard(fname)
assert shard_ntok >= num_processes * B * T + 1
ntok_total += int(shard_ntok)
self.ntok_total = ntok_total
# kick things off
self.reset()
def reset(self):
self.current_shard = 0
self.current_position = self.process_rank * self.B * self.T
self.tokens = _load_data_shard(self.files[self.current_shard])
def advance(self): # advance to next data shard
self.current_shard = (self.current_shard + 1) % len(self.files)
self.current_position = self.process_rank * self.B * self.T
self.tokens = _load_data_shard(self.files[self.current_shard])
def next_batch(self):
B = self.B
T = self.T
buf = self.tokens[self.current_position : self.current_position+B*T+1]
buf = torch.tensor(buf.astype(np.int32), dtype=torch.long)
x = (buf[:-1]).view(B, T) # inputs
y = (buf[1:]).view(B, T) # targets
# advance current position and load next shard if necessary
self.current_position += B * T * self.num_processes
if self.current_position + (B * T * self.num_processes + 1) > len(self.tokens):
self.advance()
return x.cuda(), y.cuda()
# -----------------------------------------------------------------------------
# int main
@dataclass
class Hyperparameters:
# data hyperparams
input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
# optimization hyperparams
batch_size : int = 8*64 # batch size, in sequences, across all devices
device_batch_size : int = 64 # batch size, in sequences, per device
sequence_length : int = 1024 # sequence length, in tokens
num_iterations : int = 5100 # number of iterations to run
learning_rate : float = 0.0036
warmup_iters : int = 0
warmdown_iters : int = 1450 # number of iterations of linear warmup/warmdown for triangular or trapezoidal schedule
weight_decay : float = 0
# evaluation and logging hyperparams
val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()
# set up DDP (distributed data parallel). torchrun sets this env variable
assert torch.cuda.is_available()
dist.init_process_group(backend='nccl')
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
device = f'cuda:{ddp_local_rank}'
torch.cuda.set_device(device)
print(f"using device: {device}")
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.
# convenience variables
B, T = args.device_batch_size, args.sequence_length
# calculate the number of steps to take in the val loop.
assert args.val_tokens % (B * T * ddp_world_size) == 0
val_steps = args.val_tokens // (B * T * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (B * ddp_world_size) == 0
train_accumulation_steps = args.batch_size // (B * ddp_world_size)
# load tokens
train_loader = DistributedDataLoader(args.input_bin, B, T, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, B, T, ddp_rank, ddp_world_size)
if master_process:
print(f"Training DataLoader: total number of tokens: {train_loader.ntok_total} across {len(train_loader.files)} files")
print(f"Validation DataLoader: total number of tokens: {val_loader.ntok_total} across {len(val_loader.files)} files")
x, y = train_loader.next_batch()
# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, n_layer=12, n_head=6, n_embd=768))
model = model.cuda()
if hasattr(config, "coordinate_descent_tuning"):
config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank])
raw_model = model.module # always contains the "raw" unwrapped model
ctx = torch.amp.autocast(device_type='cuda', dtype=torch.bfloat16)
# init the optimizer(s)
optimizer1 = torch.optim.AdamW(raw_model.lm_head.parameters(), lr=args.learning_rate, betas=(0.9, 0.95),
weight_decay=args.weight_decay, fused=True)
optimizer2 = Muon(raw_model.transformer.h.parameters(), lr=0.1*args.learning_rate, momentum=0.95)
optimizers = [optimizer1, optimizer2]
# learning rate decay scheduler (linear warmup and warmdown)
def get_lr(it):
assert it <= args.num_iterations
# 1) linear warmup for warmup_iters steps
if it < args.warmup_iters:
return (it+1) / args.warmup_iters
# 2) constant lr for a while
elif it < args.num_iterations - args.warmdown_iters:
return 1.0
# 3) linear warmdown
else:
decay_ratio = (args.num_iterations - it) / args.warmdown_iters
return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]
# begin logging
if master_process:
run_id = str(uuid.uuid4())
logdir = 'logs/%s/' % run_id
os.makedirs(logdir, exist_ok=True)
logfile = 'logs/%s.txt' % run_id
# create the log file
with open(logfile, "w") as f:
# begin the log by printing this file (the Python code)
f.write('='*100 + '\n')
f.write(code)
f.write('='*100 + '\n')
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
f.write(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:\n")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
f.write(f'{result.stdout}\n')
f.write('='*100 + '\n')
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.time()
# begin training
train_loader.reset()
for step in range(args.num_iterations + 1):
last_step = (step == args.num_iterations)
# This effectively ignores timing first 10 steps, which are slower for weird reasons.
# Alternately, and slightly more correctly in terms of benchmarking, we could do 10
# steps with dummy data first, and then re-initialize the model and reset the loader.
if step == 10:
training_time_ms = 0
t0 = time.time()
timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val
# once in a while evaluate the validation dataset
if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
# stop the clock
torch.cuda.synchronize()
training_time_ms += 1000 * (time.time() - t0)
# run validation batches
model.eval()
val_loader.reset()
val_loss = 0.0
for _ in range(val_steps):
x_val, y_val = val_loader.next_batch()
with ctx: # of course, we'd like to use no_grad() here too, but that creates a torch.compile error for some reason
_, loss = model(x_val, y_val, return_logits=False)
val_loss += loss.detach()
del loss
dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
val_loss /= val_steps
# log val loss to console and to logfile
if master_process:
print(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
with open(logfile, "a") as f:
f.write(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms\n')
# start the clock again
torch.cuda.synchronize()
t0 = time.time()
if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
# stop the clock
torch.cuda.synchronize()
training_time_ms += 1000 * (time.time() - t0)
# save the state of the training process
log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
# start the clock again
torch.cuda.synchronize()
t0 = time.time()
# bit confusing: we want to make sure to eval on 0th iteration
# but also after the very last iteration. so we loop for step <= num_iterations
# instead of just < num_iterations (one extra due to <=), only to do
# the validation/sampling one last time, and then we break right here as we're done.
if last_step:
break
# --------------- TRAINING SECTION BEGIN -----------------
model.train()
for i in range(1, train_accumulation_steps+1):
# forward pass
with ctx:
_, loss = model(x, y, return_logits=False)
train_loss = loss.detach()
# advance the dataset for the next batch
x, y = train_loader.next_batch()
# backward pass
if i < train_accumulation_steps:
with model.no_sync(): # there's no need to sync gradients every accumulation step
loss.backward()
else:
loss.backward() # just sync on the last step
for p in model.parameters():
p.grad /= train_accumulation_steps
# step the optimizers and schedulers
for opt, sched in zip(optimizers, schedulers):
opt.step()
sched.step()
# null the gradients
model.zero_grad(set_to_none=True)
# --------------- TRAINING SECTION END -------------------
# everything that follows now is just diagnostics, prints, logging, etc.
#dist.all_reduce(train_loss, op=dist.ReduceOp.AVG) # all-reducing the training loss would be more correct in terms of logging, but slower
if master_process:
approx_time = training_time_ms + 1000 * (time.time() - t0)
print(f"step:{step+1}/{args.num_iterations} train_loss:{train_loss.item():.4f} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")
with open(logfile, "a") as f:
f.write(f"step:{step+1}/{args.num_iterations} train_loss:{train_loss.item():.4f} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms\n")
if master_process:
print(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")
# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()
====================================================================================================
Running pytorch 2.4.1+cu121 compiled for CUDA 12.1
nvidia-smi:
Mon Oct 14 05:01:30 2024
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.129.03 Driver Version: 535.129.03 CUDA Version: 12.2 |
|-----------------------------------------+----------------------+----------------------+
| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|=========================================+======================+======================|
| 0 NVIDIA H100 80GB HBM3 On | 00000000:61:00.0 Off | 0 |
| N/A 26C P0 113W / 700W | 5787MiB / 81559MiB | 1% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
| 1 NVIDIA H100 80GB HBM3 On | 00000000:62:00.0 Off | 0 |
| N/A 32C P0 121W / 700W | 5835MiB / 81559MiB | 4% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
| 2 NVIDIA H100 80GB HBM3 On | 00000000:63:00.0 Off | 0 |
| N/A 35C P0 115W / 700W | 5835MiB / 81559MiB | 7% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
| 3 NVIDIA H100 80GB HBM3 On | 00000000:64:00.0 Off | 0 |
| N/A 31C P0 116W / 700W | 5835MiB / 81559MiB | 6% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
| 4 NVIDIA H100 80GB HBM3 On | 00000000:6A:00.0 Off | 0 |
| N/A 28C P0 118W / 700W | 5835MiB / 81559MiB | 4% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
| 5 NVIDIA H100 80GB HBM3 On | 00000000:6B:00.0 Off | 0 |
| N/A 33C P0 115W / 700W | 5835MiB / 81559MiB | 4% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
| 6 NVIDIA H100 80GB HBM3 On | 00000000:6C:00.0 Off | 0 |
| N/A 31C P0 115W / 700W | 5835MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
| 7 NVIDIA H100 80GB HBM3 On | 00000000:6D:00.0 Off | 0 |
| N/A 27C P0 117W / 700W | 5595MiB / 81559MiB | 2% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
+---------------------------------------------------------------------------------------+
| Processes: |
| GPU GI CI PID Type Process name GPU Memory |
| ID ID Usage |
|=======================================================================================|
| 0 N/A N/A 67786 C /usr/bin/python3 5774MiB |
| 1 N/A N/A 67787 C /usr/bin/python3 5822MiB |
| 2 N/A N/A 67788 C /usr/bin/python3 5822MiB |
| 3 N/A N/A 67789 C /usr/bin/python3 5822MiB |
| 4 N/A N/A 67790 C /usr/bin/python3 5822MiB |
| 5 N/A N/A 67791 C /usr/bin/python3 5822MiB |
| 6 N/A N/A 67792 C /usr/bin/python3 5822MiB |
| 7 N/A N/A 67793 C /usr/bin/python3 5582MiB |
+---------------------------------------------------------------------------------------+
====================================================================================================
step:0/5100 val_loss:16.0058 train_time:278ms step_avg:nanms
step:1/5100 train_loss:16.0026 train_time:42703ms step_avg:nanms
step:2/5100 train_loss:9.4947 train_time:53466ms step_avg:nanms
step:3/5100 train_loss:8.7243 train_time:53645ms step_avg:nanms
step:4/5100 train_loss:8.0034 train_time:53821ms step_avg:nanms
step:5/5100 train_loss:7.5714 train_time:53997ms step_avg:nanms
step:6/5100 train_loss:7.5361 train_time:54176ms step_avg:nanms
step:7/5100 train_loss:7.4515 train_time:54354ms step_avg:nanms
step:8/5100 train_loss:7.4935 train_time:54529ms step_avg:nanms
step:9/5100 train_loss:7.2292 train_time:54704ms step_avg:nanms
step:10/5100 train_loss:7.0049 train_time:54881ms step_avg:nanms
step:11/5100 train_loss:6.9134 train_time:176ms step_avg:nanms
step:12/5100 train_loss:6.8756 train_time:353ms step_avg:nanms
step:13/5100 train_loss:6.7598 train_time:530ms step_avg:176.69ms
step:14/5100 train_loss:6.6413 train_time:706ms step_avg:176.50ms
step:15/5100 train_loss:6.6229 train_time:883ms step_avg:176.64ms
step:16/5100 train_loss:6.5247 train_time:1059ms step_avg:176.49ms
step:17/5100 train_loss:6.5430 train_time:1236ms step_avg:176.50ms
step:18/5100 train_loss:6.5634 train_time:1411ms step_avg:176.33ms
step:19/5100 train_loss:6.3690 train_time:1588ms step_avg:176.40ms
step:20/5100 train_loss:6.4064 train_time:1763ms step_avg:176.34ms
step:21/5100 train_loss:6.0726 train_time:1940ms step_avg:176.34ms
step:22/5100 train_loss:6.4316 train_time:2116ms step_avg:176.31ms
step:23/5100 train_loss:6.6346 train_time:2292ms step_avg:176.29ms
step:24/5100 train_loss:6.3251 train_time:2469ms step_avg:176.38ms
step:25/5100 train_loss:6.4596 train_time:2645ms step_avg:176.34ms
step:26/5100 train_loss:6.1703 train_time:2821ms step_avg:176.33ms
step:27/5100 train_loss:6.0790 train_time:2997ms step_avg:176.30ms
step:28/5100 train_loss:6.2160 train_time:3173ms step_avg:176.28ms
step:29/5100 train_loss:5.9012 train_time:3350ms step_avg:176.31ms
step:30/5100 train_loss:6.1927 train_time:3526ms step_avg:176.30ms
step:31/5100 train_loss:6.0178 train_time:3703ms step_avg:176.34ms
step:32/5100 train_loss:5.9916 train_time:3880ms step_avg:176.36ms
step:33/5100 train_loss:5.8184 train_time:4056ms step_avg:176.37ms
step:34/5100 train_loss:6.0974 train_time:4233ms step_avg:176.37ms
step:35/5100 train_loss:6.0359 train_time:4409ms step_avg:176.37ms
step:36/5100 train_loss:6.1705 train_time:4585ms step_avg:176.35ms
step:37/5100 train_loss:6.1132 train_time:4762ms step_avg:176.38ms
step:38/5100 train_loss:6.0114 train_time:4941ms step_avg:176.48ms
step:39/5100 train_loss:5.8895 train_time:5117ms step_avg:176.44ms
step:40/5100 train_loss:5.9137 train_time:5293ms step_avg:176.44ms
step:41/5100 train_loss:5.8300 train_time:5469ms step_avg:176.42ms
step:42/5100 train_loss:5.8464 train_time:5645ms step_avg:176.41ms
step:43/5100 train_loss:5.7293 train_time:5821ms step_avg:176.40ms
step:44/5100 train_loss:5.8348 train_time:5998ms step_avg:176.42ms
step:45/5100 train_loss:5.7872 train_time:6175ms step_avg:176.42ms
step:46/5100 train_loss:5.9451 train_time:6351ms step_avg:176.43ms
step:47/5100 train_loss:5.7408 train_time:6528ms step_avg:176.42ms
step:48/5100 train_loss:5.6171 train_time:6704ms step_avg:176.42ms
step:49/5100 train_loss:5.8214 train_time:6881ms step_avg:176.43ms
step:50/5100 train_loss:5.7003 train_time:7057ms step_avg:176.43ms
step:51/5100 train_loss:5.8409 train_time:7234ms step_avg:176.43ms
step:52/5100 train_loss:5.7024 train_time:7410ms step_avg:176.43ms
step:53/5100 train_loss:5.5714 train_time:7587ms step_avg:176.44ms
step:54/5100 train_loss:5.7099 train_time:7764ms step_avg:176.46ms
step:55/5100 train_loss:5.5848 train_time:7940ms step_avg:176.44ms
step:56/5100 train_loss:5.9282 train_time:8116ms step_avg:176.43ms
step:57/5100 train_loss:5.5857 train_time:8292ms step_avg:176.42ms
step:58/5100 train_loss:5.4541 train_time:8468ms step_avg:176.42ms
step:59/5100 train_loss:5.5927 train_time:8645ms step_avg:176.43ms
step:60/5100 train_loss:5.5636 train_time:8821ms step_avg:176.42ms
step:61/5100 train_loss:5.6773 train_time:8998ms step_avg:176.43ms
step:62/5100 train_loss:5.4317 train_time:9175ms step_avg:176.44ms
step:63/5100 train_loss:5.5337 train_time:9352ms step_avg:176.45ms
step:64/5100 train_loss:5.5094 train_time:9529ms step_avg:176.46ms
step:65/5100 train_loss:5.1816 train_time:9706ms step_avg:176.47ms
step:66/5100 train_loss:5.3273 train_time:9882ms step_avg:176.47ms
step:67/5100 train_loss:5.4873 train_time:10059ms step_avg:176.48ms
step:68/5100 train_loss:5.3574 train_time:10236ms step_avg:176.48ms
step:69/5100 train_loss:5.6205 train_time:10413ms step_avg:176.49ms
step:70/5100 train_loss:5.2609 train_time:10590ms step_avg:176.50ms
step:71/5100 train_loss:5.2774 train_time:10765ms step_avg:176.48ms
step:72/5100 train_loss:5.4959 train_time:10942ms step_avg:176.49ms
step:73/5100 train_loss:5.4203 train_time:11123ms step_avg:176.55ms
step:74/5100 train_loss:5.3024 train_time:11296ms step_avg:176.51ms
step:75/5100 train_loss:5.4277 train_time:11473ms step_avg:176.51ms
step:76/5100 train_loss:5.3970 train_time:11649ms step_avg:176.51ms
step:77/5100 train_loss:5.3551 train_time:11826ms step_avg:176.50ms
step:78/5100 train_loss:5.4435 train_time:12002ms step_avg:176.51ms
step:79/5100 train_loss:5.5162 train_time:12179ms step_avg:176.51ms
step:80/5100 train_loss:5.2904 train_time:12356ms step_avg:176.51ms
step:81/5100 train_loss:5.4086 train_time:12533ms step_avg:176.53ms
step:82/5100 train_loss:5.1663 train_time:12709ms step_avg:176.52ms
step:83/5100 train_loss:5.3478 train_time:12885ms step_avg:176.51ms
step:84/5100 train_loss:5.2974 train_time:13062ms step_avg:176.52ms
step:85/5100 train_loss:5.2750 train_time:13240ms step_avg:176.53ms
step:86/5100 train_loss:5.1372 train_time:13416ms step_avg:176.52ms
step:87/5100 train_loss:5.3540 train_time:13592ms step_avg:176.52ms
step:88/5100 train_loss:5.2620 train_time:13768ms step_avg:176.51ms
step:89/5100 train_loss:5.3097 train_time:13944ms step_avg:176.51ms
step:90/5100 train_loss:5.2766 train_time:14120ms step_avg:176.51ms
step:91/5100 train_loss:5.2022 train_time:14297ms step_avg:176.50ms
step:92/5100 train_loss:5.1902 train_time:14475ms step_avg:176.52ms
step:93/5100 train_loss:5.3196 train_time:14649ms step_avg:176.50ms
step:94/5100 train_loss:5.1340 train_time:14825ms step_avg:176.49ms
step:95/5100 train_loss:5.1428 train_time:15001ms step_avg:176.49ms
step:96/5100 train_loss:5.1916 train_time:15177ms step_avg:176.48ms
step:97/5100 train_loss:5.1030 train_time:15353ms step_avg:176.48ms
step:98/5100 train_loss:5.1782 train_time:15530ms step_avg:176.47ms
step:99/5100 train_loss:5.1015 train_time:15706ms step_avg:176.47ms
step:100/5100 train_loss:5.2243 train_time:15882ms step_avg:176.47ms
step:101/5100 train_loss:5.1945 train_time:16059ms step_avg:176.47ms
step:102/5100 train_loss:5.1008 train_time:16235ms step_avg:176.47ms
step:103/5100 train_loss:5.1931 train_time:16411ms step_avg:176.47ms
step:104/5100 train_loss:5.1403 train_time:16587ms step_avg:176.46ms
step:105/5100 train_loss:4.9992 train_time:16763ms step_avg:176.46ms
step:106/5100 train_loss:5.1001 train_time:16939ms step_avg:176.45ms
step:107/5100 train_loss:5.3160 train_time:17115ms step_avg:176.44ms
step:108/5100 train_loss:5.0735 train_time:17293ms step_avg:176.45ms
step:109/5100 train_loss:4.8698 train_time:17469ms step_avg:176.45ms
step:110/5100 train_loss:5.0438 train_time:17645ms step_avg:176.45ms
step:111/5100 train_loss:5.0285 train_time:17821ms step_avg:176.45ms
step:112/5100 train_loss:4.9934 train_time:17997ms step_avg:176.44ms
step:113/5100 train_loss:5.1061 train_time:18173ms step_avg:176.44ms
step:114/5100 train_loss:5.0397 train_time:18349ms step_avg:176.43ms
step:115/5100 train_loss:4.8932 train_time:18525ms step_avg:176.43ms
step:116/5100 train_loss:5.0457 train_time:18701ms step_avg:176.43ms
step:117/5100 train_loss:4.9561 train_time:18878ms step_avg:176.43ms
step:118/5100 train_loss:4.9087 train_time:19055ms step_avg:176.43ms
step:119/5100 train_loss:5.0531 train_time:19231ms step_avg:176.44ms
step:120/5100 train_loss:5.0061 train_time:19408ms step_avg:176.43ms
step:121/5100 train_loss:4.9401 train_time:19584ms step_avg:176.43ms
step:122/5100 train_loss:4.8356 train_time:19760ms step_avg:176.43ms
step:123/5100 train_loss:4.9557 train_time:19936ms step_avg:176.43ms
step:124/5100 train_loss:4.8063 train_time:20113ms step_avg:176.43ms
step:125/5100 train_loss:5.1188 train_time:20289ms step_avg:176.42ms
step:125/5100 val_loss:4.9493 train_time:20289ms step_avg:176.43ms
step:126/5100 train_loss:4.9946 train_time:20480ms step_avg:176.55ms
step:127/5100 train_loss:4.9437 train_time:20656ms step_avg:176.55ms
step:128/5100 train_loss:4.9967 train_time:20833ms step_avg:176.55ms
step:129/5100 train_loss:4.8718 train_time:21009ms step_avg:176.55ms
step:130/5100 train_loss:5.1766 train_time:21185ms step_avg:176.54ms
step:131/5100 train_loss:4.9408 train_time:21362ms step_avg:176.54ms
step:132/5100 train_loss:4.9384 train_time:21539ms step_avg:176.55ms
step:133/5100 train_loss:4.8975 train_time:21715ms step_avg:176.54ms
step:134/5100 train_loss:4.9352 train_time:21891ms step_avg:176.54ms
step:135/5100 train_loss:4.8185 train_time:22068ms step_avg:176.54ms
step:136/5100 train_loss:4.9446 train_time:22244ms step_avg:176.54ms
step:137/5100 train_loss:4.7276 train_time:22420ms step_avg:176.53ms
step:138/5100 train_loss:4.8822 train_time:22596ms step_avg:176.53ms
step:139/5100 train_loss:4.8319 train_time:22772ms step_avg:176.53ms
step:140/5100 train_loss:4.8668 train_time:22948ms step_avg:176.52ms
step:141/5100 train_loss:4.9399 train_time:23124ms step_avg:176.52ms
step:142/5100 train_loss:4.8072 train_time:23300ms step_avg:176.51ms
step:143/5100 train_loss:4.8598 train_time:23477ms step_avg:176.52ms
step:144/5100 train_loss:4.7252 train_time:23652ms step_avg:176.51ms
step:145/5100 train_loss:4.8631 train_time:23828ms step_avg:176.50ms
step:146/5100 train_loss:4.8110 train_time:24004ms step_avg:176.50ms
step:147/5100 train_loss:4.6902 train_time:24181ms step_avg:176.51ms
step:148/5100 train_loss:4.8383 train_time:24358ms step_avg:176.51ms
step:149/5100 train_loss:4.8407 train_time:24535ms step_avg:176.51ms
step:150/5100 train_loss:4.8537 train_time:24711ms step_avg:176.51ms
step:151/5100 train_loss:4.8991 train_time:24890ms step_avg:176.52ms
step:152/5100 train_loss:4.7800 train_time:25065ms step_avg:176.51ms
step:153/5100 train_loss:4.7851 train_time:25241ms step_avg:176.51ms
step:154/5100 train_loss:4.8742 train_time:25418ms step_avg:176.51ms
step:155/5100 train_loss:4.8277 train_time:25594ms step_avg:176.51ms
step:156/5100 train_loss:4.7787 train_time:25770ms step_avg:176.51ms
step:157/5100 train_loss:4.8128 train_time:25946ms step_avg:176.50ms
step:158/5100 train_loss:4.9299 train_time:26122ms step_avg:176.50ms
step:159/5100 train_loss:4.7138 train_time:26298ms step_avg:176.50ms
step:160/5100 train_loss:4.7836 train_time:26474ms step_avg:176.49ms
step:161/5100 train_loss:4.6138 train_time:26649ms step_avg:176.48ms
step:162/5100 train_loss:4.8028 train_time:26826ms step_avg:176.48ms
step:163/5100 train_loss:4.8306 train_time:27010ms step_avg:176.53ms
step:164/5100 train_loss:4.8191 train_time:27186ms step_avg:176.53ms
step:165/5100 train_loss:4.6339 train_time:27361ms step_avg:176.52ms
step:166/5100 train_loss:4.7510 train_time:27537ms step_avg:176.52ms
step:167/5100 train_loss:4.8860 train_time:27713ms step_avg:176.51ms
step:168/5100 train_loss:4.6828 train_time:27889ms step_avg:176.51ms
step:169/5100 train_loss:4.7675 train_time:28065ms step_avg:176.51ms
step:170/5100 train_loss:4.6191 train_time:28242ms step_avg:176.52ms
step:171/5100 train_loss:4.5174 train_time:28418ms step_avg:176.51ms
step:172/5100 train_loss:4.6884 train_time:28594ms step_avg:176.51ms
step:173/5100 train_loss:4.6659 train_time:28771ms step_avg:176.51ms
step:174/5100 train_loss:4.7133 train_time:28947ms step_avg:176.50ms
step:175/5100 train_loss:4.8760 train_time:29122ms step_avg:176.50ms
step:176/5100 train_loss:4.7220 train_time:29298ms step_avg:176.49ms
step:177/5100 train_loss:4.5810 train_time:29474ms step_avg:176.49ms
step:178/5100 train_loss:4.5401 train_time:29650ms step_avg:176.49ms
step:179/5100 train_loss:4.6193 train_time:29827ms step_avg:176.49ms
step:180/5100 train_loss:4.6132 train_time:30002ms step_avg:176.48ms
step:181/5100 train_loss:4.6202 train_time:30178ms step_avg:176.48ms
step:182/5100 train_loss:4.7540 train_time:30355ms step_avg:176.48ms
step:183/5100 train_loss:4.6130 train_time:30532ms step_avg:176.49ms
step:184/5100 train_loss:4.5672 train_time:30708ms step_avg:176.48ms
step:185/5100 train_loss:4.5755 train_time:30884ms step_avg:176.48ms
step:186/5100 train_loss:4.7012 train_time:31060ms step_avg:176.48ms
step:187/5100 train_loss:4.6133 train_time:31236ms step_avg:176.48ms
step:188/5100 train_loss:4.7916 train_time:31412ms step_avg:176.47ms
step:189/5100 train_loss:4.6349 train_time:31819ms step_avg:177.76ms
step:190/5100 train_loss:4.5500 train_time:32239ms step_avg:179.10ms
step:191/5100 train_loss:4.6854 train_time:32414ms step_avg:179.09ms
step:192/5100 train_loss:4.5359 train_time:32592ms step_avg:179.07ms
step:193/5100 train_loss:4.4593 train_time:32769ms step_avg:179.06ms
step:194/5100 train_loss:4.6817 train_time:32945ms step_avg:179.05ms
step:195/5100 train_loss:4.6099 train_time:33121ms step_avg:179.03ms
step:196/5100 train_loss:4.8048 train_time:33297ms step_avg:179.02ms
step:197/5100 train_loss:4.6597 train_time:33473ms step_avg:179.00ms
step:198/5100 train_loss:4.5105 train_time:33649ms step_avg:178.98ms
step:199/5100 train_loss:4.5878 train_time:33825ms step_avg:178.97ms
step:200/5100 train_loss:4.4466 train_time:34001ms step_avg:178.95ms
step:201/5100 train_loss:4.5474 train_time:34177ms step_avg:178.94ms
step:202/5100 train_loss:4.4441 train_time:34353ms step_avg:178.92ms
step:203/5100 train_loss:4.6859 train_time:34529ms step_avg:178.91ms
step:204/5100 train_loss:4.5510 train_time:34705ms step_avg:178.89ms
step:205/5100 train_loss:4.5904 train_time:34882ms step_avg:178.88ms
step:206/5100 train_loss:4.7008 train_time:35058ms step_avg:178.87ms
step:207/5100 train_loss:4.3588 train_time:35234ms step_avg:178.85ms
step:208/5100 train_loss:4.5183 train_time:35410ms step_avg:178.84ms
step:209/5100 train_loss:4.4942 train_time:35586ms step_avg:178.82ms
step:210/5100 train_loss:4.6552 train_time:35762ms step_avg:178.81ms
step:211/5100 train_loss:4.5780 train_time:35937ms step_avg:178.79ms
step:212/5100 train_loss:4.4570 train_time:36114ms step_avg:178.78ms
step:213/5100 train_loss:4.5755 train_time:36290ms step_avg:178.77ms
step:214/5100 train_loss:4.4309 train_time:36466ms step_avg:178.75ms
step:215/5100 train_loss:4.5025 train_time:36642ms step_avg:178.74ms
step:216/5100 train_loss:4.3593 train_time:36817ms step_avg:178.72ms
step:217/5100 train_loss:4.4584 train_time:36993ms step_avg:178.71ms
step:218/5100 train_loss:4.4218 train_time:37168ms step_avg:178.69ms
step:219/5100 train_loss:4.4525 train_time:37344ms step_avg:178.68ms
step:220/5100 train_loss:4.4454 train_time:37521ms step_avg:178.67ms
step:221/5100 train_loss:4.4802 train_time:37696ms step_avg:178.65ms
step:222/5100 train_loss:4.4963 train_time:37873ms step_avg:178.64ms
step:223/5100 train_loss:4.4229 train_time:38048ms step_avg:178.63ms
step:224/5100 train_loss:4.4148 train_time:38224ms step_avg:178.62ms
step:225/5100 train_loss:4.6404 train_time:38399ms step_avg:178.60ms
step:226/5100 train_loss:4.2775 train_time:38575ms step_avg:178.59ms
step:227/5100 train_loss:4.3405 train_time:38751ms step_avg:178.58ms
step:228/5100 train_loss:4.3456 train_time:38928ms step_avg:178.57ms
step:229/5100 train_loss:4.5040 train_time:39104ms step_avg:178.56ms
step:230/5100 train_loss:4.2919 train_time:39280ms step_avg:178.55ms
step:231/5100 train_loss:4.4256 train_time:39458ms step_avg:178.54ms
step:232/5100 train_loss:4.2853 train_time:39634ms step_avg:178.53ms
step:233/5100 train_loss:4.3117 train_time:39809ms step_avg:178.52ms
step:234/5100 train_loss:4.4765 train_time:39987ms step_avg:178.51ms
step:235/5100 train_loss:4.3544 train_time:40162ms step_avg:178.50ms
step:236/5100 train_loss:4.2542 train_time:40338ms step_avg:178.49ms
step:237/5100 train_loss:4.4520 train_time:40514ms step_avg:178.48ms
step:238/5100 train_loss:4.4211 train_time:40691ms step_avg:178.47ms
step:239/5100 train_loss:4.2868 train_time:40866ms step_avg:178.45ms
step:240/5100 train_loss:4.4460 train_time:41042ms step_avg:178.44ms
step:241/5100 train_loss:4.4481 train_time:41218ms step_avg:178.44ms
step:242/5100 train_loss:4.3234 train_time:41394ms step_avg:178.42ms
step:243/5100 train_loss:4.5057 train_time:41570ms step_avg:178.41ms
step:244/5100 train_loss:4.3363 train_time:41745ms step_avg:178.40ms
step:245/5100 train_loss:4.3795 train_time:41921ms step_avg:178.39ms
step:246/5100 train_loss:4.4645 train_time:42097ms step_avg:178.38ms
step:247/5100 train_loss:4.3934 train_time:42273ms step_avg:178.37ms
step:248/5100 train_loss:4.3306 train_time:42448ms step_avg:178.35ms
step:249/5100 train_loss:4.4495 train_time:42624ms step_avg:178.34ms
step:250/5100 train_loss:4.2343 train_time:42800ms step_avg:178.33ms
step:250/5100 val_loss:4.3294 train_time:42801ms step_avg:178.34ms
step:251/5100 train_loss:4.2866 train_time:42990ms step_avg:178.38ms
step:252/5100 train_loss:4.3910 train_time:43167ms step_avg:178.38ms
step:253/5100 train_loss:4.4375 train_time:43343ms step_avg:178.37ms
step:254/5100 train_loss:4.2618 train_time:43519ms step_avg:178.36ms
step:255/5100 train_loss:4.2056 train_time:43695ms step_avg:178.35ms
step:256/5100 train_loss:4.3795 train_time:43870ms step_avg:178.33ms
step:257/5100 train_loss:4.2962 train_time:44045ms step_avg:178.32ms
step:258/5100 train_loss:4.3110 train_time:44221ms step_avg:178.31ms
step:259/5100 train_loss:4.2766 train_time:44397ms step_avg:178.30ms
step:260/5100 train_loss:4.3112 train_time:44573ms step_avg:178.29ms
step:261/5100 train_loss:4.3578 train_time:44749ms step_avg:178.28ms
step:262/5100 train_loss:4.3110 train_time:44924ms step_avg:178.27ms
step:263/5100 train_loss:4.2845 train_time:45100ms step_avg:178.26ms
step:264/5100 train_loss:4.1933 train_time:45276ms step_avg:178.25ms
step:265/5100 train_loss:4.2789 train_time:45452ms step_avg:178.24ms
step:266/5100 train_loss:4.1411 train_time:45628ms step_avg:178.23ms
step:267/5100 train_loss:4.2057 train_time:45804ms step_avg:178.23ms
step:268/5100 train_loss:4.2190 train_time:45980ms step_avg:178.22ms
step:269/5100 train_loss:4.2343 train_time:46156ms step_avg:178.21ms
step:270/5100 train_loss:4.1531 train_time:46332ms step_avg:178.20ms
step:271/5100 train_loss:4.3739 train_time:46508ms step_avg:178.19ms
step:272/5100 train_loss:4.2763 train_time:46685ms step_avg:178.19ms
step:273/5100 train_loss:4.1906 train_time:46861ms step_avg:178.18ms
step:274/5100 train_loss:4.2347 train_time:47037ms step_avg:178.17ms
step:275/5100 train_loss:4.3184 train_time:47212ms step_avg:178.16ms
step:276/5100 train_loss:4.3326 train_time:47389ms step_avg:178.15ms
step:277/5100 train_loss:4.5033 train_time:47565ms step_avg:178.15ms
step:278/5100 train_loss:4.3076 train_time:47741ms step_avg:178.14ms
step:279/5100 train_loss:4.3757 train_time:47917ms step_avg:178.13ms
step:280/5100 train_loss:4.2652 train_time:48093ms step_avg:178.12ms
step:281/5100 train_loss:4.3849 train_time:48269ms step_avg:178.11ms
step:282/5100 train_loss:4.2231 train_time:48445ms step_avg:178.11ms
step:283/5100 train_loss:4.2406 train_time:48621ms step_avg:178.10ms
step:284/5100 train_loss:4.1802 train_time:48798ms step_avg:178.09ms
step:285/5100 train_loss:4.3255 train_time:48974ms step_avg:178.09ms
step:286/5100 train_loss:4.3297 train_time:49149ms step_avg:178.08ms
step:287/5100 train_loss:4.3605 train_time:49325ms step_avg:178.07ms
step:288/5100 train_loss:4.1870 train_time:49502ms step_avg:178.06ms
step:289/5100 train_loss:4.2773 train_time:49677ms step_avg:178.05ms
step:290/5100 train_loss:4.1463 train_time:49854ms step_avg:178.05ms
step:291/5100 train_loss:4.1352 train_time:50030ms step_avg:178.04ms
step:292/5100 train_loss:4.2215 train_time:50206ms step_avg:178.04ms
step:293/5100 train_loss:4.1353 train_time:50382ms step_avg:178.03ms
step:294/5100 train_loss:4.1765 train_time:50558ms step_avg:178.02ms
step:295/5100 train_loss:4.2147 train_time:50736ms step_avg:178.02ms
step:296/5100 train_loss:4.1041 train_time:50911ms step_avg:178.01ms
step:297/5100 train_loss:4.1091 train_time:51088ms step_avg:178.01ms
step:298/5100 train_loss:4.1188 train_time:51264ms step_avg:178.00ms
step:299/5100 train_loss:4.2266 train_time:51439ms step_avg:177.99ms
step:300/5100 train_loss:4.0911 train_time:51617ms step_avg:177.99ms
step:301/5100 train_loss:4.2290 train_time:51792ms step_avg:177.98ms
step:302/5100 train_loss:4.2452 train_time:51968ms step_avg:177.97ms
step:303/5100 train_loss:4.1818 train_time:52144ms step_avg:177.97ms
step:304/5100 train_loss:4.2385 train_time:52320ms step_avg:177.96ms
step:305/5100 train_loss:4.2183 train_time:52496ms step_avg:177.95ms
step:306/5100 train_loss:4.6906 train_time:52671ms step_avg:177.94ms
step:307/5100 train_loss:4.1859 train_time:52847ms step_avg:177.94ms
step:308/5100 train_loss:4.0954 train_time:53023ms step_avg:177.93ms
step:309/5100 train_loss:4.2516 train_time:53200ms step_avg:177.93ms
step:310/5100 train_loss:4.1107 train_time:53376ms step_avg:177.92ms
step:311/5100 train_loss:4.3280 train_time:53552ms step_avg:177.91ms
step:312/5100 train_loss:4.1847 train_time:53728ms step_avg:177.91ms
step:313/5100 train_loss:4.1178 train_time:53904ms step_avg:177.90ms
step:314/5100 train_loss:4.2313 train_time:54080ms step_avg:177.89ms
step:315/5100 train_loss:4.3307 train_time:54256ms step_avg:177.89ms
step:316/5100 train_loss:4.2040 train_time:54433ms step_avg:177.88ms
step:317/5100 train_loss:4.0369 train_time:54608ms step_avg:177.88ms
step:318/5100 train_loss:4.1194 train_time:54784ms step_avg:177.87ms
step:319/5100 train_loss:4.1512 train_time:54960ms step_avg:177.86ms
step:320/5100 train_loss:4.1312 train_time:55136ms step_avg:177.86ms
step:321/5100 train_loss:4.2382 train_time:55312ms step_avg:177.85ms
step:322/5100 train_loss:4.1943 train_time:55487ms step_avg:177.84ms
step:323/5100 train_loss:4.1569 train_time:55663ms step_avg:177.84ms
step:324/5100 train_loss:4.2454 train_time:55840ms step_avg:177.83ms
step:325/5100 train_loss:4.2094 train_time:56015ms step_avg:177.83ms
step:326/5100 train_loss:4.2689 train_time:56191ms step_avg:177.82ms
step:327/5100 train_loss:4.1268 train_time:56368ms step_avg:177.82ms
step:328/5100 train_loss:4.6237 train_time:56545ms step_avg:177.81ms
step:329/5100 train_loss:4.3142 train_time:56722ms step_avg:177.81ms
step:330/5100 train_loss:4.0510 train_time:56898ms step_avg:177.81ms
step:331/5100 train_loss:3.9927 train_time:57075ms step_avg:177.80ms
step:332/5100 train_loss:4.2155 train_time:57252ms step_avg:177.80ms
step:333/5100 train_loss:4.1338 train_time:57427ms step_avg:177.79ms
step:334/5100 train_loss:4.1184 train_time:57603ms step_avg:177.79ms
step:335/5100 train_loss:4.0777 train_time:57780ms step_avg:177.78ms
step:336/5100 train_loss:4.2559 train_time:57956ms step_avg:177.78ms
step:337/5100 train_loss:4.1938 train_time:58132ms step_avg:177.77ms
step:338/5100 train_loss:4.6657 train_time:58308ms step_avg:177.77ms
step:339/5100 train_loss:4.1724 train_time:58484ms step_avg:177.76ms
step:340/5100 train_loss:4.1241 train_time:58661ms step_avg:177.76ms
step:341/5100 train_loss:4.1550 train_time:58837ms step_avg:177.76ms
step:342/5100 train_loss:4.0779 train_time:59014ms step_avg:177.75ms
step:343/5100 train_loss:4.0418 train_time:59190ms step_avg:177.75ms
step:344/5100 train_loss:4.0895 train_time:59366ms step_avg:177.74ms
step:345/5100 train_loss:4.2267 train_time:59542ms step_avg:177.74ms
step:346/5100 train_loss:4.0679 train_time:59719ms step_avg:177.73ms
step:347/5100 train_loss:4.0054 train_time:59895ms step_avg:177.73ms
step:348/5100 train_loss:4.0440 train_time:60071ms step_avg:177.73ms
step:349/5100 train_loss:4.0900 train_time:60248ms step_avg:177.72ms
step:350/5100 train_loss:4.0462 train_time:60423ms step_avg:177.72ms
step:351/5100 train_loss:3.7708 train_time:60599ms step_avg:177.71ms
step:352/5100 train_loss:4.0472 train_time:60776ms step_avg:177.71ms
step:353/5100 train_loss:4.3921 train_time:60952ms step_avg:177.70ms
step:354/5100 train_loss:3.8966 train_time:61129ms step_avg:177.70ms
step:355/5100 train_loss:4.1555 train_time:61305ms step_avg:177.70ms
step:356/5100 train_loss:4.0261 train_time:61481ms step_avg:177.69ms
step:357/5100 train_loss:4.1168 train_time:61657ms step_avg:177.69ms
step:358/5100 train_loss:4.0748 train_time:61833ms step_avg:177.68ms
step:359/5100 train_loss:4.0650 train_time:62009ms step_avg:177.68ms
step:360/5100 train_loss:4.1344 train_time:62185ms step_avg:177.67ms
step:361/5100 train_loss:3.6909 train_time:62362ms step_avg:177.67ms
step:362/5100 train_loss:4.2503 train_time:62537ms step_avg:177.66ms
step:363/5100 train_loss:4.1436 train_time:62712ms step_avg:177.66ms
step:364/5100 train_loss:4.0631 train_time:62888ms step_avg:177.65ms
step:365/5100 train_loss:3.9781 train_time:63065ms step_avg:177.65ms
step:366/5100 train_loss:4.1396 train_time:63241ms step_avg:177.64ms
step:367/5100 train_loss:4.0982 train_time:63416ms step_avg:177.64ms
step:368/5100 train_loss:4.0841 train_time:63593ms step_avg:177.63ms
step:369/5100 train_loss:4.0658 train_time:63768ms step_avg:177.63ms
step:370/5100 train_loss:3.9644 train_time:63944ms step_avg:177.62ms
step:371/5100 train_loss:4.1065 train_time:64120ms step_avg:177.62ms
step:372/5100 train_loss:3.9905 train_time:64296ms step_avg:177.61ms
step:373/5100 train_loss:3.9210 train_time:64472ms step_avg:177.61ms
step:374/5100 train_loss:4.1341 train_time:64648ms step_avg:177.60ms
step:375/5100 train_loss:4.0597 train_time:64824ms step_avg:177.60ms
step:375/5100 val_loss:4.0562 train_time:64824ms step_avg:177.60ms
step:376/5100 train_loss:4.0301 train_time:65010ms step_avg:177.62ms
step:377/5100 train_loss:4.0905 train_time:65187ms step_avg:177.62ms
step:378/5100 train_loss:4.0025 train_time:65595ms step_avg:178.25ms
step:379/5100 train_loss:4.0635 train_time:65772ms step_avg:178.24ms
step:380/5100 train_loss:4.1011 train_time:66192ms step_avg:178.90ms
step:381/5100 train_loss:4.1625 train_time:66369ms step_avg:178.89ms
step:382/5100 train_loss:4.0703 train_time:66544ms step_avg:178.88ms
step:383/5100 train_loss:4.0429 train_time:66720ms step_avg:178.87ms
step:384/5100 train_loss:4.0037 train_time:66896ms step_avg:178.87ms
step:385/5100 train_loss:4.0878 train_time:67072ms step_avg:178.86ms
step:386/5100 train_loss:3.9992 train_time:67248ms step_avg:178.85ms
step:387/5100 train_loss:4.1152 train_time:67425ms step_avg:178.85ms
step:388/5100 train_loss:4.3059 train_time:67601ms step_avg:178.84ms
step:389/5100 train_loss:4.0138 train_time:67776ms step_avg:178.83ms
step:390/5100 train_loss:4.0023 train_time:67953ms step_avg:178.82ms
step:391/5100 train_loss:4.1095 train_time:68131ms step_avg:178.82ms
step:392/5100 train_loss:4.0304 train_time:68307ms step_avg:178.81ms
step:393/5100 train_loss:4.1374 train_time:68484ms step_avg:178.81ms
step:394/5100 train_loss:3.9696 train_time:68659ms step_avg:178.80ms
step:395/5100 train_loss:4.1109 train_time:68835ms step_avg:178.79ms
step:396/5100 train_loss:3.8514 train_time:69010ms step_avg:178.78ms
step:397/5100 train_loss:4.0552 train_time:69188ms step_avg:178.78ms
step:398/5100 train_loss:4.1052 train_time:69363ms step_avg:178.77ms
step:399/5100 train_loss:4.1100 train_time:69539ms step_avg:178.76ms
step:400/5100 train_loss:4.0020 train_time:69715ms step_avg:178.76ms
step:401/5100 train_loss:4.0727 train_time:69891ms step_avg:178.75ms
step:402/5100 train_loss:4.1263 train_time:70066ms step_avg:178.74ms
step:403/5100 train_loss:4.0611 train_time:70243ms step_avg:178.73ms
step:404/5100 train_loss:4.1727 train_time:70418ms step_avg:178.73ms
step:405/5100 train_loss:3.9276 train_time:70594ms step_avg:178.72ms
step:406/5100 train_loss:4.0151 train_time:70771ms step_avg:178.71ms
step:407/5100 train_loss:4.2963 train_time:70946ms step_avg:178.71ms
step:408/5100 train_loss:4.0113 train_time:71122ms step_avg:178.70ms
step:409/5100 train_loss:4.0379 train_time:71298ms step_avg:178.69ms
step:410/5100 train_loss:4.0852 train_time:71474ms step_avg:178.68ms
step:411/5100 train_loss:3.9602 train_time:71650ms step_avg:178.68ms
step:412/5100 train_loss:3.9804 train_time:71826ms step_avg:178.67ms
step:413/5100 train_loss:4.4052 train_time:72002ms step_avg:178.67ms
step:414/5100 train_loss:3.8528 train_time:72178ms step_avg:178.66ms
step:415/5100 train_loss:4.2238 train_time:72354ms step_avg:178.65ms
step:416/5100 train_loss:3.9783 train_time:72530ms step_avg:178.65ms
step:417/5100 train_loss:3.9719 train_time:72706ms step_avg:178.64ms
step:418/5100 train_loss:4.1763 train_time:72882ms step_avg:178.63ms
step:419/5100 train_loss:3.9039 train_time:73060ms step_avg:178.63ms