bdcab079-8761-4bb8-b2bc-adc2a45fbc9a.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
from dataclasses import dataclass
from pathlib import Path

import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
# Use of FlexAttention contributed by @KoszarskyB
from torch.nn.attention.flex_attention import BlockMask, flex_attention

# -----------------------------------------------------------------------------
# Muon optimizer

def zeropower_via_svd(G, steps=None):
    U, S, V = G.svd()
    return U @ V.T

@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert len(G.shape) == 2
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    X /= (X.norm() + eps) # ensure top singular value <= 1
    if G.size(0) > G.size(1):
        X = X.T
    for _ in range(steps):
        A = X @ X.T
        B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    if G.size(0) > G.size(1):
        X = X.T
    return X

zeropower_backends = dict(svd=zeropower_via_svd, newtonschulz5=zeropower_via_newtonschulz5)

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven't tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        backend: The chosen backend for the orthogonalization step. (recommended: 'newtonschulz5')
        backend_steps: The number of iteration steps to use in the backend, if it is iterative.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True,
                 backend='newtonschulz5', backend_steps=5):
        self.num_process = int(os.environ['WORLD_SIZE'])
        self.rank = int(os.environ["RANK"])
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, backend=backend, backend_steps=backend_steps)
        params: "list[torch.Tensor]" = list(params)
        assert all(isinstance(p, torch.Tensor) for p in params)
        sizes = {p.numel() for p in params}
        param_groups = [
            {
                "params": [p for p in params if p.numel() == size],
                "update_buffer": [
                    torch.empty(size, device="cuda", dtype=torch.bfloat16)
                    for _ in range(self.num_process)
                ],
            }
            for size in sizes
        ]
        super().__init__(param_groups, defaults)

    def step(self):
        for group in self.param_groups:
            lr: float = group["lr"]
            momentum: float = group["momentum"]
            nesterov: bool = group["nesterov"]
            zeropower_backend = zeropower_backends[group["backend"]]
            backend_steps: int = group["backend_steps"]
            update_buffers: "list[torch.Tensor]" = group["update_buffer"]
            # generate weight updates in distributed fashion
            params: "list[torch.Tensor]" = group["params"]
            assert len(params) % self.num_process == 0
            handle = None
            params_world = None
            def update_prev():
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffers):
                    p_world.data.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.num_process]:
                p = params[base_i + self.rank]
                g = p.grad
                assert g is not None
                state = self.state[p] 
                if "momentum_buffer" not in state:
                    state["momentum_buffer"] = torch.zeros_like(g)
                buf: torch.Tensor = state["momentum_buffer"]
                buf.lerp_(g, 1 - momentum)
                g = g.lerp_(buf, momentum) if nesterov else buf
                g = zeropower_backend(g, steps=backend_steps).flatten()
                update_prev()
                handle = dist.all_gather(update_buffers, g, async_op=True)
                params_world = params[base_i : base_i + self.num_process]
            update_prev()


# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):

    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=False)

    def forward(self, x):
        return F.linear(x, self.weight.to(x.dtype))

class Rotary(torch.nn.Module):

    def __init__(self, dim, base=10000):
        super().__init__()
        self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None

    def forward(self, x):
        seq_len = x.shape[1]
        if seq_len != self.seq_len_cached:
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.outer(t, self.inv_freq)
            self.seq_len_cached = seq_len
            self.cos_cached = freqs.cos()
            self.sin_cached = freqs.sin()
        cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
        # apply_rotary_emb(x, cos, sin)
        x1, x2 = x.chunk(2, dim=3)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x)

class CausalSelfAttention(nn.Module):

    def __init__(self, dim, n_head):
        super().__init__()
        assert dim % n_head == 0
        self.n_head = n_head
        self.c_q = CastedLinear(dim, dim)
        self.c_k = CastedLinear(dim, dim)
        self.c_v = CastedLinear(dim, dim)
        # value residual lambda
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5])) # @Grad62304977
        # rotary embeddings
        self.rotary = Rotary(dim // n_head) # dim // n_head = head_dim
        # output projection
        self.c_proj = CastedLinear(dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor, vi: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q: torch.Tensor = self.c_q(x).view(B, T, self.n_head, -1)
        k: torch.Tensor = self.c_k(x).view(B, T, self.n_head, -1)
        v: torch.Tensor = self.c_v(x).view(B, T, self.n_head, -1)
        v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @Grad62304977
        q, k = norm(q), norm(k) # QK norm suggested by @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask)
        y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):

    def __init__(self, dim: int):
        super().__init__()
        self.c_fc   = CastedLinear(dim, 4 * dim)
        self.c_proj = CastedLinear(4 * dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):

    def __init__(self, config):
        super().__init__()
        self.attn = CausalSelfAttention(config.n_embd, config.n_head)
        self.mlp = MLP(config.n_embd)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x: torch.Tensor, vi: torch.Tensor, x0: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        x = x + self.attn(norm(x), vi, block_mask)
        x = x + self.mlp(norm(x))
        return x

# -----------------------------------------------------------------------------
# The main GPT-2 model

@dataclass
class GPTConfig:
    vocab_size : int = 50304
    n_layer : int = 12
    n_head : int = 6 # head dim 128 suggested by @Grad62304977
    n_embd : int = 768
    lm_head_softcap : int = 30

class GPT(nn.Module):

    def __init__(self, config: GPTConfig):
        super().__init__()
        self.n_layer = config.n_layer
        self.lm_head_softcap = config.lm_head_softcap

        # U-net design by @brendanh0gan
        self.num_encoder_layers = config.n_layer // 2 # Half of the layers for encoder
        self.num_decoder_layers = config.n_layer - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))

        self.transformer = nn.ModuleDict(dict(
            wte = nn.Embedding(config.vocab_size, config.n_embd),
            # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
            # U-net structure on token value embeddings by @leloykun
            vte = nn.Embedding(config.vocab_size, config.n_embd*self.num_encoder_layers),
            h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
        ))
        self.lm_head = CastedLinear(config.n_embd, config.vocab_size)
        self.lm_head.weight.data.zero_() # @Grad62304977

    def forward(self, idx: torch.Tensor, target: torch.Tensor, sliding_window: torch.Tensor) -> torch.Tensor:
        BLOCK_SIZE = 128
        assert idx.ndim == 1
        docs = (idx == 50256).cumsum(0)
        docs_low = docs.reshape(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.reshape(-1, BLOCK_SIZE)[:, -1].contiguous()
        def document_sliding_window_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            window_mask = q_idx - kv_idx < sliding_window
            return causal_mask & document_mask & window_mask

        S = len(idx)
        def create_sliding_window_causal_mask(S: int, sliding_window: torch.Tensor):
            kv_idx = block_idx = torch.arange(S // BLOCK_SIZE, dtype=torch.int32, device="cuda")
            q_idx = block_idx[:, None]
            causal_mask = q_idx >= kv_idx
            document_mask = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
            window_mask = q_idx - kv_idx < ((sliding_window + BLOCK_SIZE - 1) // BLOCK_SIZE)
            dense_mask = causal_mask & document_mask & window_mask
            dense_mask = dense_mask.to(torch.int32)
            num_blocks = dense_mask.sum(dim=-1).to(torch.int32)
            indices = torch.argsort(dense_mask, dim=-1, descending=True, stable=True).to(torch.int32)
            num_blocks = num_blocks[None, None, :].contiguous()
            indices = indices[None, None, :].contiguous()
            return BlockMask.from_kv_blocks(num_blocks, indices, BLOCK_SIZE=BLOCK_SIZE, mask_mod=document_sliding_window_causal)
        block_mask = create_sliding_window_causal_mask(S, sliding_window)

        # forward the GPT model itself
        x = self.transformer.wte(idx[None]) # token embeddings of shape (b, t, n_embd)
        x = norm(x) # @Grad62304977
        x0 = x
        vi = self.transformer.vte(idx[None]).chunk(self.num_encoder_layers, dim=-1)

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        for i in range(self.num_encoder_layers):
            x = self.transformer.h[i](x, vi[i], x0, block_mask)
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            # U-net structure on token value embeddings by @leloykun
            x = self.transformer.h[self.num_encoder_layers + i](x, vi[self.num_encoder_layers-1-i], x0, block_mask)

        x = norm(x)
        logits = self.lm_head(x)
        logits = self.lm_head_softcap * torch.tanh(logits / self.lm_head_softcap) # @Grad62304977
        logits = logits.float()
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), target.view(-1))
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _peek_data_shard(file: Path):
    # only reads the header, returns header data
    # header is 256 int32
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    return int(header[2]) # number of tokens (claimed)

def _load_data_shard(file: Path, ntok: int):
    with file.open("rb") as f:
        tokens = torch.empty(ntok, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy())
        assert nbytes == 2 * ntok, "number of tokens read does not match header?"
    return tokens

class DistributedDataLoader:
    def __init__(self, filename_pattern, T, process_rank, num_processes):
        self.process_rank = process_rank
        self.num_processes = num_processes
        self.T = T

        # glob files that match the pattern
        self.files = sorted(Path.cwd().glob(filename_pattern))
        assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"

        # load and validate all data shards, count number of tokens in total
        self.ntoks = [_peek_data_shard(file) for file in self.files]
        assert min(self.ntoks) >= num_processes * T + 1
        self.ntok_total = sum(self.ntoks)

        self.reset()

    def reset(self):
        self.current_shard = -1
        self.advance()

    def advance(self): # advance to next data shard
        self.current_shard = (self.current_shard + 1) % len(self.files)
        self.current_position = self.process_rank * self.T
        self.tokens = _load_data_shard(self.files[self.current_shard], self.ntoks[self.current_shard])

    def next_batch(self):
        batch_size = self.T * self.num_processes
        buf = self.tokens[self.current_position:self.current_position+self.T+1]
        # host side async is sufficient;
        # no performance improvement was observed when introducing a separate stream.
        x = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
        y = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
        # advance current position and load next shard if necessary
        self.current_position += batch_size
        if self.current_position + batch_size + 1 >= len(self.tokens):
            self.advance()
        return x, y

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data hyperparams
    input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
    input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
    # optimization hyperparams
    batch_size : int = 8 # batch size, in sequences, across all devices
    sequence_length : int = 64*1024 # sequence length, in tokens
    num_iterations : int = 1480 # number of iterations to run
    warmup_iters : int = 0
    cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
    weight_decay : float = 0
    # evaluation and logging hyperparams
    val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
    val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()

# set up DDP (distributed data parallel). torchrun sets this env variable
assert torch.cuda.is_available()
dist.init_process_group(backend='nccl')
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
device = f'cuda:{ddp_local_rank}'
torch.cuda.set_device(device)
print(f"using device: {device}")
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = str(uuid.uuid4())
    logdir = 'logs/%s/' % run_id
    # os.makedirs(logdir, exist_ok=True)
    logfile = 'logs/%s.txt' % run_id
    # create the log file
    with open(logfile, "w") as f:
        # begin the log by printing this file (the Python code)
        f.write(code)
        f.write('='*100 + '\n')
def print0(s, logonly=False):
    if master_process:
        with open(logfile, "a") as f:
            if not logonly:
                print(s)
            f.write(s+'\n')
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)

# convenience variables
T = args.sequence_length
# calculate the number of steps to take in the val loop.
assert args.val_tokens % (T * ddp_world_size) == 0
val_steps = args.val_tokens // (T * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size
assert train_accumulation_steps == 1

# load tokens
train_loader = DistributedDataLoader(args.input_bin, T, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, T, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.ntok_total} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.ntok_total} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
x, y = train_loader.next_batch()

# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, n_layer=12, n_head=6, n_embd=768))
model = model.cuda().bfloat16()
for m in model.modules():
    if isinstance(m, CastedLinear):
        m.float()
if hasattr(config, "coordinate_descent_tuning"):
    config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank])
raw_model = model.module # always contains the "raw" unwrapped model

# init the optimizer(s)
optimizer1 = torch.optim.Adam([raw_model.transformer.wte.weight, raw_model.transformer.vte.weight], lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.transformer.h.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
    assert it <= args.num_iterations
    # 1) linear warmup for warmup_iters steps
    if it < args.warmup_iters:
        return (it+1) / args.warmup_iters
    # 2) constant lr for a while
    elif it < args.num_iterations - args.cooldown_iters:
        return 1.0
    # 3) linear cooldown
    else:
        decay_ratio = (args.num_iterations - it) / args.cooldown_iters
        return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]

sliding_window_size = torch.tensor(64, dtype=torch.int32, device="cuda")
sw_size_prev = 64
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
    last_step = (step == args.num_iterations)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Set the sliding window size for the current step, in chunks of 64. By @fernbear.bsky.social
    sw_size =  64 * int((64 + (1792 - 64) * step / args.num_iterations) // 64)
    if sw_size != sw_size_prev:
        sliding_window_size.copy_(sw_size, non_blocking=True)
        sw_size_prev = sw_size

    # once in a while evaluate the validation dataset
    if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # run validation batches
        model.eval()
        val_loader.reset()
        val_loss = 0.0
        for _ in range(val_steps):
            with torch.no_grad():
                x_val, y_val = val_loader.next_batch()
                val_loss += model(x_val, y_val, sliding_window=sliding_window_size)
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        val_loss /= val_steps
        # log val loss to console and to logfile
        print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # save the state of the training process
        log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
        # torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    # bit confusing: we want to make sure to eval on 0th iteration
    # but also after the very last iteration. so we loop for step <= num_iterations
    # instead of just < num_iterations (one extra due to <=), only to do
    # the validation/sampling one last time, and then we break right here as we're done.
    if last_step:
        break

    # --------------- TRAINING SECTION BEGIN -----------------
    model.train()
    loss = model(x, y, sliding_window=sliding_window_size)
    loss.backward()
    del loss
    # advance the dataset for the next batch
    x, y = train_loader.next_batch()
    # momentum warmup for Muon
    frac = min(step/300, 1)
    for group in optimizer3.param_groups:
        group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # --------------- TRAINING SECTION END -------------------
    # everything that follows now is just diagnostics, prints, logging, etc.
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")

if master_process:
    print(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")

# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()
====================================================================================================
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Sun Dec  8 11:03:47 2024       
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.6     |
|-----------------------------------------+----------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
|                                         |                      |               MIG M. |
|=========================================+======================+======================|
|   0  NVIDIA H100 80GB HBM3          On  | 00000000:65:02.0 Off |                    0 |
| N/A   36C    P0              73W / 700W |      7MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  | 00000000:67:02.0 Off |                    0 |
| N/A   46C    P0             130W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  | 00000000:69:02.0 Off |                    0 |
| N/A   45C    P0             123W / 700W |    533MiB / 81559MiB |      1%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  | 00000000:6B:02.0 Off |                    0 |
| N/A   39C    P0             100W / 700W |     27MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  | 00000000:6F:02.0 Off |                    0 |
| N/A   39C    P0             105W / 700W |    533MiB / 81559MiB |      1%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  | 00000000:71:02.0 Off |                    0 |
| N/A   45C    P0             117W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  | 00000000:73:02.0 Off |                    0 |
| N/A   46C    P0              95W / 700W |     26MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  | 00000000:75:02.0 Off |                    0 |
| N/A   38C    P0              99W / 700W |     26MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
                                                                                         
+---------------------------------------------------------------------------------------+
| Processes:                                                                            |
|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
|        ID   ID                                                             Usage      |
|=======================================================================================|
+---------------------------------------------------------------------------------------+

====================================================================================================
Training DataLoader: total number of tokens: 3200000000 across 32 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:23618ms step_avg:nanms
step:2/1480 train_time:23712ms step_avg:nanms
step:3/1480 train_time:23852ms step_avg:nanms
step:4/1480 train_time:23993ms step_avg:nanms
step:5/1480 train_time:24134ms step_avg:nanms
step:6/1480 train_time:24275ms step_avg:nanms
step:7/1480 train_time:24417ms step_avg:nanms
step:8/1480 train_time:24558ms step_avg:nanms
step:9/1480 train_time:24703ms step_avg:nanms
step:10/1480 train_time:24847ms step_avg:nanms
step:11/1480 train_time:142ms step_avg:nanms
step:12/1480 train_time:286ms step_avg:nanms
step:13/1480 train_time:427ms step_avg:142.18ms
step:14/1480 train_time:568ms step_avg:142.03ms
step:15/1480 train_time:709ms step_avg:141.78ms
step:16/1480 train_time:852ms step_avg:142.06ms
step:17/1480 train_time:998ms step_avg:142.55ms
step:18/1480 train_time:1141ms step_avg:142.66ms
step:19/1480 train_time:1285ms step_avg:142.79ms
step:20/1480 train_time:1427ms step_avg:142.75ms
step:21/1480 train_time:1569ms step_avg:142.60ms
step:22/1480 train_time:1710ms step_avg:142.47ms
step:23/1480 train_time:1851ms step_avg:142.40ms
step:24/1480 train_time:1995ms step_avg:142.48ms
step:25/1480 train_time:2140ms step_avg:142.69ms
step:26/1480 train_time:2285ms step_avg:142.80ms
step:27/1480 train_time:2428ms step_avg:142.80ms
step:28/1480 train_time:2570ms step_avg:142.77ms
step:29/1480 train_time:2711ms step_avg:142.68ms
step:30/1480 train_time:2853ms step_avg:142.63ms
step:31/1480 train_time:2996ms step_avg:142.67ms
step:32/1480 train_time:3141ms step_avg:142.75ms
step:33/1480 train_time:3285ms step_avg:142.81ms
step:34/1480 train_time:3428ms step_avg:142.83ms
step:35/1480 train_time:3569ms step_avg:142.78ms
step:36/1480 train_time:3711ms step_avg:142.72ms
step:37/1480 train_time:3851ms step_avg:142.63ms
step:38/1480 train_time:3994ms step_avg:142.63ms
step:39/1480 train_time:4138ms step_avg:142.69ms
step:40/1480 train_time:4282ms step_avg:142.75ms
step:41/1480 train_time:4426ms step_avg:142.79ms
step:42/1480 train_time:4569ms step_avg:142.78ms
step:43/1480 train_time:4710ms step_avg:142.73ms
step:44/1480 train_time:4851ms step_avg:142.68ms
step:45/1480 train_time:4992ms step_avg:142.63ms
step:46/1480 train_time:5135ms step_avg:142.65ms
step:47/1480 train_time:5280ms step_avg:142.69ms
step:48/1480 train_time:5423ms step_avg:142.70ms
step:49/1480 train_time:5566ms step_avg:142.73ms
step:50/1480 train_time:5709ms step_avg:142.72ms
step:51/1480 train_time:5849ms step_avg:142.67ms
step:52/1480 train_time:5990ms step_avg:142.63ms
step:53/1480 train_time:6133ms step_avg:142.62ms
step:54/1480 train_time:6274ms step_avg:142.60ms
step:55/1480 train_time:6418ms step_avg:142.63ms
step:56/1480 train_time:6564ms step_avg:142.69ms
step:57/1480 train_time:6708ms step_avg:142.71ms
step:58/1480 train_time:6849ms step_avg:142.69ms
step:59/1480 train_time:6990ms step_avg:142.66ms
step:60/1480 train_time:7132ms step_avg:142.65ms
step:61/1480 train_time:7275ms step_avg:142.64ms
step:62/1480 train_time:7417ms step_avg:142.63ms
step:63/1480 train_time:7560ms step_avg:142.64ms
step:64/1480 train_time:7704ms step_avg:142.67ms
step:65/1480 train_time:7847ms step_avg:142.67ms
step:66/1480 train_time:7988ms step_avg:142.65ms
step:67/1480 train_time:8131ms step_avg:142.65ms
step:68/1480 train_time:8272ms step_avg:142.62ms
step:69/1480 train_time:8413ms step_avg:142.59ms
step:70/1480 train_time:8557ms step_avg:142.61ms
step:71/1480 train_time:8701ms step_avg:142.64ms
step:72/1480 train_time:8844ms step_avg:142.65ms
step:73/1480 train_time:8987ms step_avg:142.65ms
step:74/1480 train_time:9129ms step_avg:142.64ms
step:75/1480 train_time:9271ms step_avg:142.62ms
step:76/1480 train_time:9412ms step_avg:142.60ms
step:77/1480 train_time:9553ms step_avg:142.59ms
step:78/1480 train_time:9697ms step_avg:142.60ms
step:79/1480 train_time:9841ms step_avg:142.63ms
step:80/1480 train_time:9986ms step_avg:142.65ms
step:81/1480 train_time:10128ms step_avg:142.65ms
step:82/1480 train_time:10270ms step_avg:142.64ms
step:83/1480 train_time:10411ms step_avg:142.62ms
step:84/1480 train_time:10553ms step_avg:142.60ms
step:85/1480 train_time:10694ms step_avg:142.59ms
step:86/1480 train_time:10837ms step_avg:142.59ms
step:87/1480 train_time:10980ms step_avg:142.60ms
step:88/1480 train_time:11122ms step_avg:142.60ms
step:89/1480 train_time:11265ms step_avg:142.60ms
step:90/1480 train_time:11408ms step_avg:142.60ms
step:91/1480 train_time:11549ms step_avg:142.58ms
step:92/1480 train_time:11691ms step_avg:142.58ms
step:93/1480 train_time:11835ms step_avg:142.59ms
step:94/1480 train_time:11978ms step_avg:142.60ms
step:95/1480 train_time:12122ms step_avg:142.61ms
step:96/1480 train_time:12265ms step_avg:142.62ms
step:97/1480 train_time:12408ms step_avg:142.62ms
step:98/1480 train_time:12550ms step_avg:142.61ms
step:99/1480 train_time:12693ms step_avg:142.62ms
step:100/1480 train_time:12836ms step_avg:142.62ms
step:101/1480 train_time:12980ms step_avg:142.63ms
step:102/1480 train_time:13122ms step_avg:142.63ms
step:103/1480 train_time:13265ms step_avg:142.64ms
step:104/1480 train_time:13407ms step_avg:142.62ms
step:105/1480 train_time:13548ms step_avg:142.61ms
step:106/1480 train_time:13690ms step_avg:142.60ms
step:107/1480 train_time:13832ms step_avg:142.60ms
step:108/1480 train_time:13974ms step_avg:142.59ms
step:109/1480 train_time:14115ms step_avg:142.58ms
step:110/1480 train_time:14258ms step_avg:142.58ms
step:111/1480 train_time:14404ms step_avg:142.62ms
step:112/1480 train_time:14551ms step_avg:142.66ms
step:113/1480 train_time:14696ms step_avg:142.68ms
step:114/1480 train_time:14844ms step_avg:142.73ms
step:115/1480 train_time:14991ms step_avg:142.77ms
step:116/1480 train_time:15137ms step_avg:142.80ms
step:117/1480 train_time:15286ms step_avg:142.86ms
step:118/1480 train_time:15433ms step_avg:142.90ms
step:119/1480 train_time:15580ms step_avg:142.94ms
step:120/1480 train_time:15727ms step_avg:142.97ms
step:121/1480 train_time:15872ms step_avg:142.99ms
step:122/1480 train_time:16017ms step_avg:143.01ms
step:123/1480 train_time:16165ms step_avg:143.06ms
step:124/1480 train_time:16312ms step_avg:143.09ms
step:125/1480 train_time:16460ms step_avg:143.13ms
step:125/1480 val_loss:4.4132 train_time:16517ms step_avg:143.62ms
step:126/1480 train_time:16612ms step_avg:143.21ms
step:127/1480 train_time:16761ms step_avg:143.25ms
step:128/1480 train_time:16907ms step_avg:143.28ms
step:129/1480 train_time:17053ms step_avg:143.30ms
step:130/1480 train_time:17199ms step_avg:143.33ms
step:131/1480 train_time:17345ms step_avg:143.35ms
step:132/1480 train_time:17489ms step_avg:143.36ms
step:133/1480 train_time:17638ms step_avg:143.39ms
step:134/1480 train_time:17784ms step_avg:143.42ms
step:135/1480 train_time:17930ms step_avg:143.44ms
step:136/1480 train_time:18078ms step_avg:143.47ms
step:137/1480 train_time:18225ms step_avg:143.50ms
step:138/1480 train_time:18370ms step_avg:143.52ms
step:139/1480 train_time:18518ms step_avg:143.55ms
step:140/1480 train_time:18665ms step_avg:143.58ms
step:141/1480 train_time:18810ms step_avg:143.59ms
step:142/1480 train_time:18957ms step_avg:143.61ms
step:143/1480 train_time:19104ms step_avg:143.64ms
step:144/1480 train_time:19250ms step_avg:143.66ms
step:145/1480 train_time:19399ms step_avg:143.69ms
step:146/1480 train_time:19546ms step_avg:143.72ms
step:147/1480 train_time:19693ms step_avg:143.74ms
step:148/1480 train_time:19840ms step_avg:143.77ms
step:149/1480 train_time:19986ms step_avg:143.78ms
step:150/1480 train_time:20132ms step_avg:143.80ms
step:151/1480 train_time:20279ms step_avg:143.82ms
step:152/1480 train_time:20426ms step_avg:143.85ms
step:153/1480 train_time:20573ms step_avg:143.87ms
step:154/1480 train_time:20720ms step_avg:143.89ms
step:155/1480 train_time:20866ms step_avg:143.90ms
step:156/1480 train_time:21012ms step_avg:143.92ms
step:157/1480 train_time:21158ms step_avg:143.93ms
step:158/1480 train_time:21305ms step_avg:143.95ms
step:159/1480 train_time:21451ms step_avg:143.96ms
step:160/1480 train_time:21598ms step_avg:143.99ms
step:161/1480 train_time:21746ms step_avg:144.01ms
step:162/1480 train_time:21893ms step_avg:144.03ms
step:163/1480 train_time:22041ms step_avg:144.06ms
step:164/1480 train_time:22187ms step_avg:144.07ms
step:165/1480 train_time:22334ms step_avg:144.09ms
step:166/1480 train_time:22481ms step_avg:144.11ms
step:167/1480 train_time:22628ms step_avg:144.13ms
step:168/1480 train_time:22775ms step_avg:144.15ms
step:169/1480 train_time:22921ms step_avg:144.16ms
step:170/1480 train_time:23066ms step_avg:144.17ms
step:171/1480 train_time:23215ms step_avg:144.19ms
step:172/1480 train_time:23361ms step_avg:144.20ms
step:173/1480 train_time:23507ms step_avg:144.22ms
step:174/1480 train_time:23653ms step_avg:144.23ms
step:175/1480 train_time:23802ms step_avg:144.25ms
step:176/1480 train_time:23947ms step_avg:144.26ms
step:177/1480 train_time:24095ms step_avg:144.28ms
step:178/1480 train_time:24243ms step_avg:144.30ms
step:179/1480 train_time:24390ms step_avg:144.32ms
step:180/1480 train_time:24537ms step_avg:144.33ms
step:181/1480 train_time:24683ms step_avg:144.35ms
step:182/1480 train_time:24829ms step_avg:144.35ms
step:183/1480 train_time:24975ms step_avg:144.36ms
step:184/1480 train_time:25122ms step_avg:144.38ms
step:185/1480 train_time:25268ms step_avg:144.39ms
step:186/1480 train_time:25416ms step_avg:144.41ms
step:187/1480 train_time:25562ms step_avg:144.42ms
step:188/1480 train_time:25708ms step_avg:144.43ms
step:189/1480 train_time:25856ms step_avg:144.45ms
step:190/1480 train_time:26003ms step_avg:144.46ms
step:191/1480 train_time:26149ms step_avg:144.47ms
step:192/1480 train_time:26295ms step_avg:144.48ms
step:193/1480 train_time:26442ms step_avg:144.49ms
step:194/1480 train_time:26588ms step_avg:144.50ms
step:195/1480 train_time:26736ms step_avg:144.52ms
step:196/1480 train_time:26883ms step_avg:144.53ms
step:197/1480 train_time:27028ms step_avg:144.54ms
step:198/1480 train_time:27178ms step_avg:144.56ms
step:199/1480 train_time:27325ms step_avg:144.58ms
step:200/1480 train_time:27471ms step_avg:144.58ms
step:201/1480 train_time:27618ms step_avg:144.59ms
step:202/1480 train_time:27764ms step_avg:144.61ms
step:203/1480 train_time:27910ms step_avg:144.61ms
step:204/1480 train_time:28057ms step_avg:144.63ms
step:205/1480 train_time:28205ms step_avg:144.64ms
step:206/1480 train_time:28350ms step_avg:144.64ms
step:207/1480 train_time:28498ms step_avg:144.66ms
step:208/1480 train_time:28645ms step_avg:144.67ms
step:209/1480 train_time:28792ms step_avg:144.68ms
step:210/1480 train_time:28940ms step_avg:144.70ms
step:211/1480 train_time:29086ms step_avg:144.71ms
step:212/1480 train_time:29231ms step_avg:144.71ms
step:213/1480 train_time:29378ms step_avg:144.72ms
step:214/1480 train_time:29525ms step_avg:144.73ms
step:215/1480 train_time:29670ms step_avg:144.73ms
step:216/1480 train_time:29818ms step_avg:144.75ms
step:217/1480 train_time:29965ms step_avg:144.76ms
step:218/1480 train_time:30112ms step_avg:144.77ms
step:219/1480 train_time:30260ms step_avg:144.78ms
step:220/1480 train_time:30406ms step_avg:144.79ms
step:221/1480 train_time:30554ms step_avg:144.81ms
step:222/1480 train_time:30705ms step_avg:144.84ms
step:223/1480 train_time:30856ms step_avg:144.86ms
step:224/1480 train_time:31006ms step_avg:144.89ms
step:225/1480 train_time:31155ms step_avg:144.91ms
step:226/1480 train_time:31306ms step_avg:144.93ms
step:227/1480 train_time:31456ms step_avg:144.96ms
step:228/1480 train_time:31606ms step_avg:144.98ms
step:229/1480 train_time:31756ms step_avg:145.00ms
step:230/1480 train_time:31906ms step_avg:145.03ms
step:231/1480 train_time:32056ms step_avg:145.05ms
step:232/1480 train_time:32206ms step_avg:145.07ms
step:233/1480 train_time:32356ms step_avg:145.09ms
step:234/1480 train_time:32506ms step_avg:145.12ms
step:235/1480 train_time:32656ms step_avg:145.14ms
step:236/1480 train_time:32806ms step_avg:145.16ms
step:237/1480 train_time:32955ms step_avg:145.17ms
step:238/1480 train_time:33105ms step_avg:145.20ms
step:239/1480 train_time:33254ms step_avg:145.21ms
step:240/1480 train_time:33405ms step_avg:145.24ms
step:241/1480 train_time:33555ms step_avg:145.26ms
step:242/1480 train_time:33707ms step_avg:145.29ms
step:243/1480 train_time:33855ms step_avg:145.30ms
step:244/1480 train_time:34007ms step_avg:145.33ms
step:245/1480 train_time:34157ms step_avg:145.35ms
step:246/1480 train_time:34307ms step_avg:145.37ms
step:247/1480 train_time:34457ms step_avg:145.39ms
step:248/1480 train_time:34607ms step_avg:145.41ms
step:249/1480 train_time:34758ms step_avg:145.43ms
step:250/1480 train_time:34908ms step_avg:145.45ms
step:250/1480 val_loss:4.0002 train_time:34966ms step_avg:145.69ms
step:251/1480 train_time:35064ms step_avg:145.49ms
step:252/1480 train_time:35216ms step_avg:145.52ms
step:253/1480 train_time:35365ms step_avg:145.53ms
step:254/1480 train_time:35514ms step_avg:145.55ms
step:255/1480 train_time:35663ms step_avg:145.56ms
step:256/1480 train_time:35811ms step_avg:145.57ms
step:257/1480 train_time:35963ms step_avg:145.60ms
step:258/1480 train_time:36115ms step_avg:145.63ms
step:259/1480 train_time:36267ms step_avg:145.65ms
step:260/1480 train_time:36419ms step_avg:145.68ms
step:261/1480 train_time:36569ms step_avg:145.69ms
step:262/1480 train_time:36720ms step_avg:145.71ms
step:263/1480 train_time:36870ms step_avg:145.73ms
step:264/1480 train_time:37021ms step_avg:145.75ms
step:265/1480 train_time:37172ms step_avg:145.77ms
step:266/1480 train_time:37323ms step_avg:145.79ms
step:267/1480 train_time:37473ms step_avg:145.81ms
step:268/1480 train_time:37623ms step_avg:145.83ms
step:269/1480 train_time:37773ms step_avg:145.84ms
step:270/1480 train_time:37924ms step_avg:145.86ms
step:271/1480 train_time:38074ms step_avg:145.88ms
step:272/1480 train_time:38225ms step_avg:145.90ms
step:273/1480 train_time:38376ms step_avg:145.92ms
step:274/1480 train_time:38526ms step_avg:145.93ms
step:275/1480 train_time:38677ms step_avg:145.95ms
step:276/1480 train_time:38827ms step_avg:145.97ms
step:277/1480 train_time:38977ms step_avg:145.98ms
step:278/1480 train_time:39127ms step_avg:146.00ms
step:279/1480 train_time:39278ms step_avg:146.01ms
step:280/1480 train_time:39427ms step_avg:146.03ms
step:281/1480 train_time:39578ms step_avg:146.05ms
step:282/1480 train_time:39728ms step_avg:146.06ms
step:283/1480 train_time:39879ms step_avg:146.08ms
step:284/1480 train_time:40030ms step_avg:146.09ms
step:285/1480 train_time:40181ms step_avg:146.11ms
step:286/1480 train_time:40332ms step_avg:146.13ms
step:287/1480 train_time:40482ms step_avg:146.14ms
step:288/1480 train_time:40632ms step_avg:146.16ms
step:289/1480 train_time:40783ms step_avg:146.17ms
step:290/1480 train_time:40934ms step_avg:146.19ms
step:291/1480 train_time:41084ms step_avg:146.21ms
step:292/1480 train_time:41234ms step_avg:146.22ms
step:293/1480 train_time:41384ms step_avg:146.23ms
step:294/1480 train_time:41534ms step_avg:146.25ms
step:295/1480 train_time:41684ms step_avg:146.26ms
step:296/1480 train_time:41836ms step_avg:146.28ms
step:297/1480 train_time:41986ms step_avg:146.29ms
step:298/1480 train_time:42136ms step_avg:146.30ms
step:299/1480 train_time:42286ms step_avg:146.32ms
step:300/1480 train_time:42438ms step_avg:146.34ms
step:301/1480 train_time:42587ms step_avg:146.35ms
step:302/1480 train_time:42738ms step_avg:146.36ms
step:303/1480 train_time:42888ms step_avg:146.37ms
step:304/1480 train_time:43039ms step_avg:146.39ms
step:305/1480 train_time:43189ms step_avg:146.40ms
step:306/1480 train_time:43340ms step_avg:146.42ms
step:307/1480 train_time:43490ms step_avg:146.43ms
step:308/1480 train_time:43642ms step_avg:146.45ms
step:309/1480 train_time:43790ms step_avg:146.46ms
step:310/1480 train_time:43941ms step_avg:146.47ms
step:311/1480 train_time:44091ms step_avg:146.48ms
step:312/1480 train_time:44242ms step_avg:146.50ms
step:313/1480 train_time:44393ms step_avg:146.51ms
step:314/1480 train_time:44543ms step_avg:146.52ms
step:315/1480 train_time:44693ms step_avg:146.53ms
step:316/1480 train_time:44844ms step_avg:146.55ms
step:317/1480 train_time:44994ms step_avg:146.56ms
step:318/1480 train_time:45144ms step_avg:146.57ms
step:319/1480 train_time:45292ms step_avg:146.58ms
step:320/1480 train_time:45442ms step_avg:146.59ms
step:321/1480 train_time:45594ms step_avg:146.60ms
step:322/1480 train_time:45745ms step_avg:146.62ms
step:323/1480 train_time:45898ms step_avg:146.64ms
step:324/1480 train_time:46048ms step_avg:146.65ms
step:325/1480 train_time:46199ms step_avg:146.66ms
step:326/1480 train_time:46349ms step_avg:146.67ms
step:327/1480 train_time:46499ms step_avg:146.69ms
step:328/1480 train_time:46651ms step_avg:146.70ms
step:329/1480 train_time:46802ms step_avg:146.71ms
step:330/1480 train_time:46954ms step_avg:146.73ms
step:331/1480 train_time:47108ms step_avg:146.75ms
step:332/1480 train_time:47261ms step_avg:146.77ms
step:333/1480 train_time:47415ms step_avg:146.80ms
step:334/1480 train_time:47568ms step_avg:146.82ms
step:335/1480 train_time:47722ms step_avg:146.84ms
step:336/1480 train_time:47876ms step_avg:146.86ms
step:337/1480 train_time:48031ms step_avg:146.88ms
step:338/1480 train_time:48184ms step_avg:146.90ms
step:339/1480 train_time:48336ms step_avg:146.92ms
step:340/1480 train_time:48490ms step_avg:146.94ms
step:341/1480 train_time:48644ms step_avg:146.96ms
step:342/1480 train_time:48797ms step_avg:146.98ms
step:343/1480 train_time:48951ms step_avg:147.00ms
step:344/1480 train_time:49105ms step_avg:147.02ms
step:345/1480 train_time:49261ms step_avg:147.05ms
step:346/1480 train_time:49415ms step_avg:147.07ms
step:347/1480 train_time:49567ms step_avg:147.08ms
step:348/1480 train_time:49722ms step_avg:147.11ms
step:349/1480 train_time:49876ms step_avg:147.13ms
step:350/1480 train_time:50030ms step_avg:147.15ms
step:351/1480 train_time:50185ms step_avg:147.17ms
step:352/1480 train_time:50339ms step_avg:147.19ms
step:353/1480 train_time:50494ms step_avg:147.21ms
step:354/1480 train_time:50646ms step_avg:147.23ms
step:355/1480 train_time:50799ms step_avg:147.24ms
step:356/1480 train_time:50955ms step_avg:147.27ms
step:357/1480 train_time:51109ms step_avg:147.29ms
step:358/1480 train_time:51263ms step_avg:147.31ms
step:359/1480 train_time:51416ms step_avg:147.32ms
step:360/1480 train_time:51572ms step_avg:147.35ms
step:361/1480 train_time:51727ms step_avg:147.37ms
step:362/1480 train_time:51880ms step_avg:147.39ms
step:363/1480 train_time:52033ms step_avg:147.40ms
step:364/1480 train_time:52186ms step_avg:147.42ms
step:365/1480 train_time:52340ms step_avg:147.44ms
step:366/1480 train_time:52493ms step_avg:147.45ms
step:367/1480 train_time:52647ms step_avg:147.47ms
step:368/1480 train_time:52800ms step_avg:147.49ms
step:369/1480 train_time:52955ms step_avg:147.51ms
step:370/1480 train_time:53108ms step_avg:147.52ms
step:371/1480 train_time:53263ms step_avg:147.54ms
step:372/1480 train_time:53416ms step_avg:147.56ms
step:373/1480 train_time:53569ms step_avg:147.57ms
step:374/1480 train_time:53723ms step_avg:147.59ms
step:375/1480 train_time:53876ms step_avg:147.61ms
step:375/1480 val_loss:3.8081 train_time:53938ms step_avg:147.78ms
step:376/1480 train_time:54036ms step_avg:147.64ms
step:377/1480 train_time:54192ms step_avg:147.66ms
step:378/1480 train_time:54346ms step_avg:147.68ms
step:379/1480 train_time:54498ms step_avg:147.69ms
step:380/1480 train_time:54651ms step_avg:147.71ms
step:381/1480 train_time:54803ms step_avg:147.72ms
step:382/1480 train_time:54957ms step_avg:147.73ms
step:383/1480 train_time:55112ms step_avg:147.75ms
step:384/1480 train_time:55266ms step_avg:147.77ms
step:385/1480 train_time:55420ms step_avg:147.79ms
step:386/1480 train_time:55573ms step_avg:147.80ms
step:387/1480 train_time:55728ms step_avg:147.82ms
step:388/1480 train_time:55881ms step_avg:147.83ms
step:389/1480 train_time:56035ms step_avg:147.85ms
step:390/1480 train_time:56190ms step_avg:147.87ms
step:391/1480 train_time:56344ms step_avg:147.89ms
step:392/1480 train_time:56497ms step_avg:147.90ms
step:393/1480 train_time:56652ms step_avg:147.92ms
step:394/1480 train_time:56806ms step_avg:147.93ms
step:395/1480 train_time:56959ms step_avg:147.94ms
step:396/1480 train_time:57114ms step_avg:147.96ms
step:397/1480 train_time:57269ms step_avg:147.98ms
step:398/1480 train_time:57423ms step_avg:148.00ms
step:399/1480 train_time:57575ms step_avg:148.01ms
step:400/1480 train_time:57730ms step_avg:148.03ms
step:401/1480 train_time:57885ms step_avg:148.04ms
step:402/1480 train_time:58038ms step_avg:148.06ms
step:403/1480 train_time:58191ms step_avg:148.07ms
step:404/1480 train_time:58346ms step_avg:148.09ms
step:405/1480 train_time:58500ms step_avg:148.10ms
step:406/1480 train_time:58655ms step_avg:148.12ms
step:407/1480 train_time:58808ms step_avg:148.13ms
step:408/1480 train_time:58961ms step_avg:148.14ms
step:409/1480 train_time:59115ms step_avg:148.16ms
step:410/1480 train_time:59268ms step_avg:148.17ms
step:411/1480 train_time:59422ms step_avg:148.18ms
step:412/1480 train_time:59576ms step_avg:148.20ms
step:413/1480 train_time:59728ms step_avg:148.21ms
step:414/1480 train_time:59883ms step_avg:148.23ms
step:415/1480 train_time:60037ms step_avg:148.24ms
step:416/1480 train_time:60190ms step_avg:148.25ms
step:417/1480 train_time:60346ms step_avg:148.27ms
step:418/1480 train_time:60498ms step_avg:148.28ms
step:419/1480 train_time:60652ms step_avg:148.29ms
step:420/1480 train_time:60805ms step_avg:148.30ms
step:421/1480 train_time:60958ms step_avg:148.32ms
step:422/1480 train_time:61113ms step_avg:148.33ms
step:423/1480 train_time:61269ms step_avg:148.35ms
step:424/1480 train_time:61423ms step_avg:148.37ms
step:425/1480 train_time:61577ms step_avg:148.38ms
step:426/1480 train_time:61731ms step_avg:148.39ms
step:427/1480 train_time:61885ms step_avg:148.40ms
step:428/1480 train_time:62037ms step_avg:148.41ms
step:429/1480 train_time:62191ms step_avg:148.43ms
step:430/1480 train_time:62346ms step_avg:148.44ms
step:431/1480 train_time:62500ms step_avg:148.46ms
step:432/1480 train_time:62653ms step_avg:148.47ms
step:433/1480 train_time:62807ms step_avg:148.48ms
step:434/1480 train_time:62960ms step_avg:148.49ms
step:435/1480 train_time:63115ms step_avg:148.51ms
step:436/1480 train_time:63270ms step_avg:148.52ms
step:437/1480 train_time:63424ms step_avg:148.53ms
step:438/1480 train_time:63576ms step_avg:148.54ms
step:439/1480 train_time:63731ms step_avg:148.56ms
step:440/1480 train_time:63887ms step_avg:148.57ms
step:441/1480 train_time:64044ms step_avg:148.59ms
step:442/1480 train_time:64200ms step_avg:148.61ms
step:443/1480 train_time:64355ms step_avg:148.63ms
step:444/1480 train_time:64512ms step_avg:148.65ms
step:445/1480 train_time:64668ms step_avg:148.66ms
step:446/1480 train_time:64824ms step_avg:148.68ms
step:447/1480 train_time:64979ms step_avg:148.69ms
step:448/1480 train_time:65135ms step_avg:148.71ms
step:449/1480 train_time:65293ms step_avg:148.73ms
step:450/1480 train_time:65452ms step_avg:148.76ms
step:451/1480 train_time:65611ms step_avg:148.78ms
step:452/1480 train_time:65768ms step_avg:148.80ms
step:453/1480 train_time:65925ms step_avg:148.81ms
step:454/1480 train_time:66079ms step_avg:148.83ms
step:455/1480 train_time:66235ms step_avg:148.84ms
step:456/1480 train_time:66392ms step_avg:148.86ms
step:457/1480 train_time:66549ms step_avg:148.88ms
step:458/1480 train_time:66704ms step_avg:148.89ms
step:459/1480 train_time:66862ms step_avg:148.91ms
step:460/1480 train_time:67020ms step_avg:148.93ms
step:461/1480 train_time:67179ms step_avg:148.95ms
step:462/1480 train_time:67336ms step_avg:148.97ms
step:463/1480 train_time:67493ms step_avg:148.99ms
step:464/1480 train_time:67650ms step_avg:149.01ms
step:465/1480 train_time:67808ms step_avg:149.03ms
step:466/1480 train_time:67964ms step_avg:149.04ms
step:467/1480 train_time:68123ms step_avg:149.07ms
step:468/1480 train_time:68279ms step_avg:149.08ms
step:469/1480 train_time:68435ms step_avg:149.10ms
step:470/1480 train_time:68592ms step_avg:149.11ms
step:471/1480 train_time:68750ms step_avg:149.13ms
step:472/1480 train_time:68907ms step_avg:149.15ms
step:473/1480 train_time:69063ms step_avg:149.16ms
step:474/1480 train_time:69219ms step_avg:149.18ms
step:475/1480 train_time:69375ms step_avg:149.19ms
step:476/1480 train_time:69533ms step_avg:149.21ms
step:477/1480 train_time:69690ms step_avg:149.23ms
step:478/1480 train_time:69846ms step_avg:149.24ms
step:479/1480 train_time:70003ms step_avg:149.26ms
step:480/1480 train_time:70160ms step_avg:149.28ms
step:481/1480 train_time:70318ms step_avg:149.29ms
step:482/1480 train_time:70474ms step_avg:149.31ms
step:483/1480 train_time:70631ms step_avg:149.33ms
step:484/1480 train_time:70786ms step_avg:149.34ms
step:485/1480 train_time:70943ms step_avg:149.35ms
step:486/1480 train_time:71100ms step_avg:149.37ms
step:487/1480 train_time:71258ms step_avg:149.39ms
step:488/1480 train_time:71416ms step_avg:149.41ms
step:489/1480 train_time:71573ms step_avg:149.42ms
step:490/1480 train_time:71730ms step_avg:149.44ms
step:491/1480 train_time:71887ms step_avg:149.45ms
step:492/1480 train_time:72042ms step_avg:149.47ms
step:493/1480 train_time:72200ms step_avg:149.48ms
step:494/1480 train_time:72358ms step_avg:149.50ms
step:495/1480 train_time:72515ms step_avg:149.52ms
step:496/1480 train_time:72672ms step_avg:149.53ms
step:497/1480 train_time:72828ms step_avg:149.54ms
step:498/1480 train_time:72985ms step_avg:149.56ms
step:499/1480 train_time:73141ms step_avg:149.57ms
step:500/1480 train_time:73297ms step_avg:149.59ms
step:500/1480 val_loss:3.6868 train_time:73361ms step_avg:149.72ms
step:501/1480 train_time:73459ms step_avg:149.61ms
step:502/1480 train_time:73618ms step_avg:149.63ms
step:503/1480 train_time:73774ms step_avg:149.64ms
step:504/1480 train_time:73931ms step_avg:149.66ms
step:505/1480 train_time:74086ms step_avg:149.67ms
step:506/1480 train_time:74243ms step_avg:149.68ms
step:507/1480 train_time:74402ms step_avg:149.70ms
step:508/1480 train_time:74559ms step_avg:149.72ms
step:509/1480 train_time:74717ms step_avg:149.73ms
step:510/1480 train_time:74874ms step_avg:149.75ms
step:511/1480 train_time:75032ms step_avg:149.76ms
step:512/1480 train_time:75188ms step_avg:149.78ms
step:513/1480 train_time:75344ms step_avg:149.79ms
step:514/1480 train_time:75502ms step_avg:149.81ms
step:515/1480 train_time:75659ms step_avg:149.82ms
step:516/1480 train_time:75819ms step_avg:149.84ms
step:517/1480 train_time:75977ms step_avg:149.86ms
step:518/1480 train_time:76135ms step_avg:149.87ms
step:519/1480 train_time:76293ms step_avg:149.89ms
step:520/1480 train_time:76450ms step_avg:149.90ms
step:521/1480 train_time:76607ms step_avg:149.92ms
step:522/1480 train_time:76765ms step_avg:149.93ms
step:523/1480 train_time:76922ms step_avg:149.95ms
step:524/1480 train_time:77078ms step_avg:149.96ms
step:525/1480 train_time:77237ms step_avg:149.98ms
step:526/1480 train_time:77395ms step_avg:149.99ms
step:527/1480 train_time:77551ms step_avg:150.00ms
step:528/1480 train_time:77707ms step_avg:150.01ms
step:529/1480 train_time:77864ms step_avg:150.03ms
step:530/1480 train_time:78022ms step_avg:150.04ms
step:531/1480 train_time:78179ms step_avg:150.06ms
step:532/1480 train_time:78337ms step_avg:150.07ms
step:533/1480 train_time:78496ms step_avg:150.09ms
step:534/1480 train_time:78649ms step_avg:150.09ms
step:535/1480 train_time:78805ms step_avg:150.11ms
step:536/1480 train_time:78963ms step_avg:150.12ms
step:537/1480 train_time:79120ms step_avg:150.13ms
step:538/1480 train_time:79281ms step_avg:150.15ms
step:539/1480 train_time:79437ms step_avg:150.17ms
step:540/1480 train_time:79596ms step_avg:150.18ms
step:541/1480 train_time:79752ms step_avg:150.19ms
step:542/1480 train_time:79908ms step_avg:150.20ms
step:543/1480 train_time:80063ms step_avg:150.21ms
step:544/1480 train_time:80222ms step_avg:150.23ms
step:545/1480 train_time:80378ms step_avg:150.24ms
step:546/1480 train_time:80536ms step_avg:150.25ms
step:547/1480 train_time:80692ms step_avg:150.26ms
step:548/1480 train_time:80850ms step_avg:150.28ms
step:549/1480 train_time:81006ms step_avg:150.29ms
step:550/1480 train_time:81164ms step_avg:150.30ms
step:551/1480 train_time:81321ms step_avg:150.32ms
step:552/1480 train_time:81480ms step_avg:150.33ms
step:553/1480 train_time:81640ms step_avg:150.35ms
step:554/1480 train_time:81800ms step_avg:150.37ms
step:555/1480 train_time:81960ms step_avg:150.38ms
step:556/1480 train_time:82119ms step_avg:150.40ms
step:557/1480 train_time:82279ms step_avg:150.42ms
step:558/1480 train_time:82439ms step_avg:150.44ms
step:559/1480 train_time:82599ms step_avg:150.45ms
step:560/1480 train_time:82758ms step_avg:150.47ms
step:561/1480 train_time:82917ms step_avg:150.49ms
step:562/1480 train_time:83078ms step_avg:150.50ms
step:563/1480 train_time:83238ms step_avg:150.52ms
step:564/1480 train_time:83398ms step_avg:150.54ms
step:565/1480 train_time:83557ms step_avg:150.55ms
step:566/1480 train_time:83719ms step_avg:150.57ms
step:567/1480 train_time:83879ms step_avg:150.59ms
step:568/1480 train_time:84038ms step_avg:150.61ms
step:569/1480 train_time:84198ms step_avg:150.62ms
step:570/1480 train_time:84358ms step_avg:150.64ms
step:571/1480 train_time:84518ms step_avg:150.66ms
step:572/1480 train_time:84678ms step_avg:150.67ms
step:573/1480 train_time:84838ms step_avg:150.69ms
step:574/1480 train_time:85000ms step_avg:150.71ms
step:575/1480 train_time:85160ms step_avg:150.73ms
step:576/1480 train_time:85320ms step_avg:150.74ms
step:577/1480 train_time:85479ms step_avg:150.76ms
step:578/1480 train_time:85639ms step_avg:150.77ms
step:579/1480 train_time:85799ms step_avg:150.79ms
step:580/1480 train_time:85958ms step_avg:150.80ms
step:581/1480 train_time:86119ms step_avg:150.82ms
step:582/1480 train_time:86279ms step_avg:150.84ms
step:583/1480 train_time:86440ms step_avg:150.85ms
step:584/1480 train_time:86600ms step_avg:150.87ms
step:585/1480 train_time:86759ms step_avg:150.88ms
step:586/1480 train_time:86920ms step_avg:150.90ms
step:587/1480 train_time:87080ms step_avg:150.92ms
step:588/1480 train_time:87239ms step_avg:150.93ms
step:589/1480 train_time:87399ms step_avg:150.95ms
step:590/1480 train_time:87560ms step_avg:150.96ms
step:591/1480 train_time:87718ms step_avg:150.98ms
step:592/1480 train_time:87878ms step_avg:150.99ms
step:593/1480 train_time:88040ms step_avg:151.01ms
step:594/1480 train_time:88201ms step_avg:151.03ms
step:595/1480 train_time:88362ms step_avg:151.05ms
step:596/1480 train_time:88522ms step_avg:151.06ms
step:597/1480 train_time:88680ms step_avg:151.07ms
step:598/1480 train_time:88839ms step_avg:151.09ms
step:599/1480 train_time:88999ms step_avg:151.10ms
step:600/1480 train_time:89159ms step_avg:151.12ms
step:601/1480 train_time:89320ms step_avg:151.13ms
step:602/1480 train_time:89481ms step_avg:151.15ms
step:603/1480 train_time:89640ms step_avg:151.16ms
step:604/1480 train_time:89799ms step_avg:151.18ms
step:605/1480 train_time:89958ms step_avg:151.19ms
step:606/1480 train_time:90120ms step_avg:151.21ms
step:607/1480 train_time:90281ms step_avg:151.22ms
step:608/1480 train_time:90441ms step_avg:151.24ms
step:609/1480 train_time:90600ms step_avg:151.25ms
step:610/1480 train_time:90759ms step_avg:151.26ms
step:611/1480 train_time:90920ms step_avg:151.28ms
step:612/1480 train_time:91079ms step_avg:151.29ms
step:613/1480 train_time:91241ms step_avg:151.31ms
step:614/1480 train_time:91402ms step_avg:151.33ms
step:615/1480 train_time:91560ms step_avg:151.34ms
step:616/1480 train_time:91719ms step_avg:151.35ms
step:617/1480 train_time:91879ms step_avg:151.37ms
step:618/1480 train_time:92038ms step_avg:151.38ms
step:619/1480 train_time:92199ms step_avg:151.39ms
step:620/1480 train_time:92359ms step_avg:151.41ms
step:621/1480 train_time:92520ms step_avg:151.42ms
step:622/1480 train_time:92679ms step_avg:151.44ms
step:623/1480 train_time:92841ms step_avg:151.45ms
step:624/1480 train_time:93000ms step_avg:151.47ms
step:625/1480 train_time:93159ms step_avg:151.48ms
step:625/1480 val_loss:3.6070 train_time:93223ms step_avg:151.58ms
step:626/1480 train_time:93323ms step_avg:151.50ms
step:627/1480 train_time:93483ms step_avg:151.51ms
step:628/1480 train_time:93642ms step_avg:151.52ms
step:629/1480 train_time:93801ms step_avg:151.54ms
step:630/1480 train_time:93960ms step_avg:151.55ms
step:631/1480 train_time:94118ms step_avg:151.56ms
step:632/1480 train_time:94277ms step_avg:151.57ms
step:633/1480 train_time:94436ms step_avg:151.58ms
step:634/1480 train_time:94596ms step_avg:151.60ms
step:635/1480 train_time:94755ms step_avg:151.61ms
step:636/1480 train_time:94913ms step_avg:151.62ms
step:637/1480 train_time:95072ms step_avg:151.63ms
step:638/1480 train_time:95230ms step_avg:151.64ms
step:639/1480 train_time:95388ms step_avg:151.65ms
step:640/1480 train_time:95546ms step_avg:151.66ms
step:641/1480 train_time:95707ms step_avg:151.67ms
step:642/1480 train_time:95866ms step_avg:151.69ms
step:643/1480 train_time:96026ms step_avg:151.70ms
step:644/1480 train_time:96185ms step_avg:151.71ms
step:645/1480 train_time:96344ms step_avg:151.72ms
step:646/1480 train_time:96504ms step_avg:151.74ms
step:647/1480 train_time:96664ms step_avg:151.75ms
step:648/1480 train_time:96825ms step_avg:151.76ms
step:649/1480 train_time:96986ms step_avg:151.78ms
step:650/1480 train_time:97145ms step_avg:151.79ms
step:651/1480 train_time:97306ms step_avg:151.80ms
step:652/1480 train_time:97466ms step_avg:151.82ms
step:653/1480 train_time:97625ms step_avg:151.83ms
step:654/1480 train_time:97786ms step_avg:151.84ms
step:655/1480 train_time:97945ms step_avg:151.85ms
step:656/1480 train_time:98106ms step_avg:151.87ms
step:657/1480 train_time:98266ms step_avg:151.88ms
step:658/1480 train_time:98425ms step_avg:151.89ms
step:659/1480 train_time:98588ms step_avg:151.91ms
step:660/1480 train_time:98748ms step_avg:151.92ms
step:661/1480 train_time:98911ms step_avg:151.94ms
step:662/1480 train_time:99071ms step_avg:151.95ms
step:663/1480 train_time:99230ms step_avg:151.96ms
step:664/1480 train_time:99393ms step_avg:151.98ms
step:665/1480 train_time:99554ms step_avg:151.99ms
step:666/1480 train_time:99714ms step_avg:152.00ms
step:667/1480 train_time:99877ms step_avg:152.02ms
step:668/1480 train_time:100039ms step_avg:152.03ms
step:669/1480 train_time:100201ms step_avg:152.05ms
step:670/1480 train_time:100362ms step_avg:152.06ms
step:671/1480 train_time:100522ms step_avg:152.08ms
step:672/1480 train_time:100685ms step_avg:152.09ms
step:673/1480 train_time:100848ms step_avg:152.11ms
step:674/1480 train_time:101010ms step_avg:152.12ms
step:675/1480 train_time:101171ms step_avg:152.14ms
step:676/1480 train_time:101332ms step_avg:152.15ms
step:677/1480 train_time:101493ms step_avg:152.16ms
step:678/1480 train_time:101652ms step_avg:152.17ms
step:679/1480 train_time:101813ms step_avg:152.19ms
step:680/1480 train_time:101977ms step_avg:152.20ms
step:681/1480 train_time:102137ms step_avg:152.22ms
step:682/1480 train_time:102300ms step_avg:152.23ms
step:683/1480 train_time:102462ms step_avg:152.25ms
step:684/1480 train_time:102623ms step_avg:152.26ms
step:685/1480 train_time:102788ms step_avg:152.28ms
step:686/1480 train_time:102949ms step_avg:152.29ms
step:687/1480 train_time:103109ms step_avg:152.30ms
step:688/1480 train_time:103271ms step_avg:152.32ms
step:689/1480 train_time:103434ms step_avg:152.33ms
step:690/1480 train_time:103597ms step_avg:152.35ms
step:691/1480 train_time:103759ms step_avg:152.36ms
step:692/1480 train_time:103920ms step_avg:152.38ms
step:693/1480 train_time:104083ms step_avg:152.39ms
step:694/1480 train_time:104245ms step_avg:152.41ms
step:695/1480 train_time:104407ms step_avg:152.42ms
step:696/1480 train_time:104568ms step_avg:152.43ms
step:697/1480 train_time:104731ms step_avg:152.45ms
step:698/1480 train_time:104891ms step_avg:152.46ms
step:699/1480 train_time:105053ms step_avg:152.47ms
step:700/1480 train_time:105214ms step_avg:152.48ms
step:701/1480 train_time:105374ms step_avg:152.50ms
step:702/1480 train_time:105534ms step_avg:152.51ms
step:703/1480 train_time:105694ms step_avg:152.52ms
step:704/1480 train_time:105854ms step_avg:152.53ms
step:705/1480 train_time:106016ms step_avg:152.54ms
step:706/1480 train_time:106182ms step_avg:152.56ms
step:707/1480 train_time:106344ms step_avg:152.57ms
step:708/1480 train_time:106505ms step_avg:152.59ms
step:709/1480 train_time:106667ms step_avg:152.60ms
step:710/1480 train_time:106828ms step_avg:152.61ms
step:711/1480 train_time:106990ms step_avg:152.63ms
step:712/1480 train_time:107155ms step_avg:152.64ms
step:713/1480 train_time:107317ms step_avg:152.66ms
step:714/1480 train_time:107477ms step_avg:152.67ms
step:715/1480 train_time:107637ms step_avg:152.68ms
step:716/1480 train_time:107799ms step_avg:152.69ms
step:717/1480 train_time:107964ms step_avg:152.71ms
step:718/1480 train_time:108123ms step_avg:152.72ms
step:719/1480 train_time:108284ms step_avg:152.73ms
step:720/1480 train_time:108447ms step_avg:152.74ms
step:721/1480 train_time:108609ms step_avg:152.76ms
step:722/1480 train_time:108770ms step_avg:152.77ms
step:723/1480 train_time:108931ms step_avg:152.78ms
step:724/1480 train_time:109093ms step_avg:152.79ms
step:725/1480 train_time:109256ms step_avg:152.81ms
step:726/1480 train_time:109420ms step_avg:152.82ms
step:727/1480 train_time:109584ms step_avg:152.84ms
step:728/1480 train_time:109746ms step_avg:152.85ms
step:729/1480 train_time:109908ms step_avg:152.86ms
step:730/1480 train_time:110070ms step_avg:152.88ms
step:731/1480 train_time:110230ms step_avg:152.89ms
step:732/1480 train_time:110390ms step_avg:152.90ms
step:733/1480 train_time:110552ms step_avg:152.91ms
step:734/1480 train_time:110714ms step_avg:152.92ms
step:735/1480 train_time:110874ms step_avg:152.93ms
step:736/1480 train_time:111035ms step_avg:152.94ms
step:737/1480 train_time:111199ms step_avg:152.96ms
step:738/1480 train_time:111359ms step_avg:152.97ms
step:739/1480 train_time:111519ms step_avg:152.98ms
step:740/1480 train_time:111686ms step_avg:152.99ms
step:741/1480 train_time:111849ms step_avg:153.01ms
step:742/1480 train_time:112011ms step_avg:153.02ms
step:743/1480 train_time:112172ms step_avg:153.03ms
step:744/1480 train_time:112333ms step_avg:153.04ms
step:745/1480 train_time:112498ms step_avg:153.06ms
step:746/1480 train_time:112659ms step_avg:153.07ms
step:747/1480 train_time:112819ms step_avg:153.08ms
step:748/1480 train_time:112986ms step_avg:153.10ms
step:749/1480 train_time:113150ms step_avg:153.11ms
step:750/1480 train_time:113310ms step_avg:153.12ms
step:750/1480 val_loss:3.5499 train_time:113374ms step_avg:153.21ms
step:751/1480 train_time:113476ms step_avg:153.14ms
step:752/1480 train_time:113636ms step_avg:153.15ms
step:753/1480 train_time:113796ms step_avg:153.16ms
step:754/1480 train_time:113957ms step_avg:153.17ms
step:755/1480 train_time:114118ms step_avg:153.18ms
step:756/1480 train_time:114279ms step_avg:153.19ms
step:757/1480 train_time:114447ms step_avg:153.21ms
step:758/1480 train_time:114609ms step_avg:153.22ms
step:759/1480 train_time:114773ms step_avg:153.24ms
step:760/1480 train_time:114935ms step_avg:153.25ms
step:761/1480 train_time:115097ms step_avg:153.26ms
step:762/1480 train_time:115257ms step_avg:153.27ms
step:763/1480 train_time:115417ms step_avg:153.28ms
step:764/1480 train_time:115578ms step_avg:153.29ms
step:765/1480 train_time:115741ms step_avg:153.30ms
step:766/1480 train_time:115904ms step_avg:153.31ms
step:767/1480 train_time:116066ms step_avg:153.32ms
step:768/1480 train_time:116228ms step_avg:153.34ms
step:769/1480 train_time:116392ms step_avg:153.35ms
step:770/1480 train_time:116555ms step_avg:153.36ms
step:771/1480 train_time:116718ms step_avg:153.37ms
step:772/1480 train_time:116879ms step_avg:153.38ms
step:773/1480 train_time:117044ms step_avg:153.40ms
step:774/1480 train_time:117207ms step_avg:153.41ms
step:775/1480 train_time:117371ms step_avg:153.43ms
step:776/1480 train_time:117536ms step_avg:153.44ms
step:777/1480 train_time:117700ms step_avg:153.46ms
step:778/1480 train_time:117863ms step_avg:153.47ms
step:779/1480 train_time:118025ms step_avg:153.48ms
step:780/1480 train_time:118189ms step_avg:153.49ms
step:781/1480 train_time:118353ms step_avg:153.51ms
step:782/1480 train_time:118516ms step_avg:153.52ms
step:783/1480 train_time:118677ms step_avg:153.53ms
step:784/1480 train_time:118841ms step_avg:153.54ms
step:785/1480 train_time:119003ms step_avg:153.55ms
step:786/1480 train_time:119170ms step_avg:153.57ms
step:787/1480 train_time:119334ms step_avg:153.58ms
step:788/1480 train_time:119497ms step_avg:153.60ms
step:789/1480 train_time:119658ms step_avg:153.60ms
step:790/1480 train_time:119822ms step_avg:153.62ms
step:791/1480 train_time:119988ms step_avg:153.63ms
step:792/1480 train_time:120152ms step_avg:153.65ms
step:793/1480 train_time:120314ms step_avg:153.66ms
step:794/1480 train_time:120479ms step_avg:153.67ms
step:795/1480 train_time:120646ms step_avg:153.69ms
step:796/1480 train_time:120813ms step_avg:153.71ms
step:797/1480 train_time:120977ms step_avg:153.72ms
step:798/1480 train_time:121141ms step_avg:153.73ms
step:799/1480 train_time:121308ms step_avg:153.75ms
step:800/1480 train_time:121472ms step_avg:153.76ms
step:801/1480 train_time:121636ms step_avg:153.77ms
step:802/1480 train_time:121802ms step_avg:153.79ms
step:803/1480 train_time:121965ms step_avg:153.80ms
step:804/1480 train_time:122129ms step_avg:153.81ms
step:805/1480 train_time:122293ms step_avg:153.83ms
step:806/1480 train_time:122455ms step_avg:153.84ms
step:807/1480 train_time:122616ms step_avg:153.85ms
step:808/1480 train_time:122779ms step_avg:153.86ms
step:809/1480 train_time:122941ms step_avg:153.87ms
step:810/1480 train_time:123105ms step_avg:153.88ms
step:811/1480 train_time:123268ms step_avg:153.89ms
step:812/1480 train_time:123432ms step_avg:153.91ms
step:813/1480 train_time:123592ms step_avg:153.91ms
step:814/1480 train_time:123755ms step_avg:153.92ms
step:815/1480 train_time:123918ms step_avg:153.94ms
step:816/1480 train_time:124082ms step_avg:153.95ms
step:817/1480 train_time:124246ms step_avg:153.96ms
step:818/1480 train_time:124409ms step_avg:153.97ms
step:819/1480 train_time:124573ms step_avg:153.98ms
step:820/1480 train_time:124736ms step_avg:154.00ms
step:821/1480 train_time:124896ms step_avg:154.00ms
step:822/1480 train_time:125059ms step_avg:154.01ms
step:823/1480 train_time:125221ms step_avg:154.02ms
step:824/1480 train_time:125381ms step_avg:154.03ms
step:825/1480 train_time:125546ms step_avg:154.04ms
step:826/1480 train_time:125712ms step_avg:154.06ms
step:827/1480 train_time:125877ms step_avg:154.07ms
step:828/1480 train_time:126040ms step_avg:154.08ms
step:829/1480 train_time:126203ms step_avg:154.09ms
step:830/1480 train_time:126370ms step_avg:154.11ms
step:831/1480 train_time:126535ms step_avg:154.12ms
step:832/1480 train_time:126698ms step_avg:154.13ms
step:833/1480 train_time:126862ms step_avg:154.15ms
step:834/1480 train_time:127026ms step_avg:154.16ms
step:835/1480 train_time:127191ms step_avg:154.17ms
step:836/1480 train_time:127357ms step_avg:154.19ms
step:837/1480 train_time:127519ms step_avg:154.20ms
step:838/1480 train_time:127683ms step_avg:154.21ms
step:839/1480 train_time:127846ms step_avg:154.22ms
step:840/1480 train_time:128008ms step_avg:154.23ms
step:841/1480 train_time:128171ms step_avg:154.24ms
step:842/1480 train_time:128334ms step_avg:154.25ms
step:843/1480 train_time:128496ms step_avg:154.26ms
step:844/1480 train_time:128657ms step_avg:154.27ms
step:845/1480 train_time:128821ms step_avg:154.28ms
step:846/1480 train_time:128986ms step_avg:154.29ms
step:847/1480 train_time:129151ms step_avg:154.30ms
step:848/1480 train_time:129312ms step_avg:154.31ms
step:849/1480 train_time:129475ms step_avg:154.32ms
step:850/1480 train_time:129638ms step_avg:154.33ms
step:851/1480 train_time:129804ms step_avg:154.34ms
step:852/1480 train_time:129967ms step_avg:154.36ms
step:853/1480 train_time:130130ms step_avg:154.37ms
step:854/1480 train_time:130295ms step_avg:154.38ms
step:855/1480 train_time:130458ms step_avg:154.39ms
step:856/1480 train_time:130620ms step_avg:154.40ms
step:857/1480 train_time:130785ms step_avg:154.41ms
step:858/1480 train_time:130952ms step_avg:154.42ms
step:859/1480 train_time:131116ms step_avg:154.44ms
step:860/1480 train_time:131278ms step_avg:154.44ms
step:861/1480 train_time:131445ms step_avg:154.46ms
step:862/1480 train_time:131613ms step_avg:154.48ms
step:863/1480 train_time:131781ms step_avg:154.49ms
step:864/1480 train_time:131945ms step_avg:154.50ms
step:865/1480 train_time:132108ms step_avg:154.51ms
step:866/1480 train_time:132276ms step_avg:154.53ms
step:867/1480 train_time:132438ms step_avg:154.54ms
step:868/1480 train_time:132598ms step_avg:154.54ms
step:869/1480 train_time:132760ms step_avg:154.55ms
step:870/1480 train_time:132925ms step_avg:154.56ms
step:871/1480 train_time:133088ms step_avg:154.57ms
step:872/1480 train_time:133252ms step_avg:154.58ms
step:873/1480 train_time:133414ms step_avg:154.59ms
step:874/1480 train_time:133580ms step_avg:154.61ms
step:875/1480 train_time:133744ms step_avg:154.62ms
step:875/1480 val_loss:3.5035 train_time:133809ms step_avg:154.69ms
step:876/1480 train_time:133910ms step_avg:154.63ms
step:877/1480 train_time:134076ms step_avg:154.64ms
step:878/1480 train_time:134240ms step_avg:154.65ms
step:879/1480 train_time:134404ms step_avg:154.67ms
step:880/1480 train_time:134566ms step_avg:154.67ms
step:881/1480 train_time:134728ms step_avg:154.68ms
step:882/1480 train_time:134893ms step_avg:154.69ms
step:883/1480 train_time:135058ms step_avg:154.71ms
step:884/1480 train_time:135225ms step_avg:154.72ms
step:885/1480 train_time:135390ms step_avg:154.73ms
step:886/1480 train_time:135557ms step_avg:154.74ms
step:887/1480 train_time:135724ms step_avg:154.76ms
step:888/1480 train_time:135898ms step_avg:154.78ms
step:889/1480 train_time:136066ms step_avg:154.80ms
step:890/1480 train_time:136228ms step_avg:154.80ms
step:891/1480 train_time:136394ms step_avg:154.82ms
step:892/1480 train_time:136559ms step_avg:154.83ms
step:893/1480 train_time:136721ms step_avg:154.84ms
step:894/1480 train_time:136887ms step_avg:154.85ms
step:895/1480 train_time:137055ms step_avg:154.86ms
step:896/1480 train_time:137223ms step_avg:154.88ms
step:897/1480 train_time:137390ms step_avg:154.89ms
step:898/1480 train_time:137558ms step_avg:154.91ms
step:899/1480 train_time:137722ms step_avg:154.92ms
step:900/1480 train_time:137885ms step_avg:154.93ms
step:901/1480 train_time:138052ms step_avg:154.94ms
step:902/1480 train_time:138218ms step_avg:154.95ms
step:903/1480 train_time:138389ms step_avg:154.97ms
step:904/1480 train_time:138554ms step_avg:154.98ms
step:905/1480 train_time:138717ms step_avg:154.99ms
step:906/1480 train_time:138883ms step_avg:155.00ms
step:907/1480 train_time:139051ms step_avg:155.02ms
step:908/1480 train_time:139214ms step_avg:155.03ms
step:909/1480 train_time:139378ms step_avg:155.04ms
step:910/1480 train_time:139550ms step_avg:155.06ms
step:911/1480 train_time:139716ms step_avg:155.07ms
step:912/1480 train_time:139883ms step_avg:155.08ms
step:913/1480 train_time:140051ms step_avg:155.10ms
step:914/1480 train_time:140219ms step_avg:155.11ms
step:915/1480 train_time:140387ms step_avg:155.12ms
step:916/1480 train_time:140552ms step_avg:155.13ms
step:917/1480 train_time:140716ms step_avg:155.14ms
step:918/1480 train_time:140883ms step_avg:155.16ms
step:919/1480 train_time:141053ms step_avg:155.17ms
step:920/1480 train_time:141219ms step_avg:155.19ms
step:921/1480 train_time:141383ms step_avg:155.20ms
step:922/1480 train_time:141551ms step_avg:155.21ms
step:923/1480 train_time:141713ms step_avg:155.22ms
step:924/1480 train_time:141877ms step_avg:155.23ms
step:925/1480 train_time:142042ms step_avg:155.24ms
step:926/1480 train_time:142207ms step_avg:155.25ms
step:927/1480 train_time:142371ms step_avg:155.26ms
step:928/1480 train_time:142537ms step_avg:155.27ms
step:929/1480 train_time:142703ms step_avg:155.28ms
step:930/1480 train_time:142867ms step_avg:155.29ms
step:931/1480 train_time:143031ms step_avg:155.30ms
step:932/1480 train_time:143198ms step_avg:155.31ms
step:933/1480 train_time:143364ms step_avg:155.32ms
step:934/1480 train_time:143530ms step_avg:155.34ms
step:935/1480 train_time:143702ms step_avg:155.35ms
step:936/1480 train_time:143868ms step_avg:155.36ms
step:937/1480 train_time:144039ms step_avg:155.38ms
step:938/1480 train_time:144202ms step_avg:155.39ms
step:939/1480 train_time:144372ms step_avg:155.41ms
step:940/1480 train_time:144538ms step_avg:155.42ms
step:941/1480 train_time:144703ms step_avg:155.43ms
step:942/1480 train_time:144868ms step_avg:155.44ms
step:943/1480 train_time:145039ms step_avg:155.45ms
step:944/1480 train_time:145211ms step_avg:155.47ms
step:945/1480 train_time:145376ms step_avg:155.48ms
step:946/1480 train_time:145544ms step_avg:155.50ms
step:947/1480 train_time:145712ms step_avg:155.51ms
step:948/1480 train_time:145879ms step_avg:155.52ms
step:949/1480 train_time:146043ms step_avg:155.53ms
step:950/1480 train_time:146206ms step_avg:155.54ms
step:951/1480 train_time:146375ms step_avg:155.55ms
step:952/1480 train_time:146540ms step_avg:155.56ms
step:953/1480 train_time:146709ms step_avg:155.58ms
step:954/1480 train_time:146877ms step_avg:155.59ms
step:955/1480 train_time:147040ms step_avg:155.60ms
step:956/1480 train_time:147205ms step_avg:155.61ms
step:957/1480 train_time:147373ms step_avg:155.62ms
step:958/1480 train_time:147543ms step_avg:155.64ms
step:959/1480 train_time:147706ms step_avg:155.64ms
step:960/1480 train_time:147875ms step_avg:155.66ms
step:961/1480 train_time:148040ms step_avg:155.67ms
step:962/1480 train_time:148204ms step_avg:155.68ms
step:963/1480 train_time:148369ms step_avg:155.69ms
step:964/1480 train_time:148537ms step_avg:155.70ms
step:965/1480 train_time:148702ms step_avg:155.71ms
step:966/1480 train_time:148866ms step_avg:155.72ms
step:967/1480 train_time:149031ms step_avg:155.73ms
step:968/1480 train_time:149196ms step_avg:155.74ms
step:969/1480 train_time:149361ms step_avg:155.75ms
step:970/1480 train_time:149524ms step_avg:155.75ms
step:971/1480 train_time:149688ms step_avg:155.76ms
step:972/1480 train_time:149852ms step_avg:155.77ms
step:973/1480 train_time:150018ms step_avg:155.78ms
step:974/1480 train_time:150186ms step_avg:155.79ms
step:975/1480 train_time:150350ms step_avg:155.80ms
step:976/1480 train_time:150517ms step_avg:155.81ms
step:977/1480 train_time:150682ms step_avg:155.82ms
step:978/1480 train_time:150847ms step_avg:155.83ms
step:979/1480 train_time:151014ms step_avg:155.84ms
step:980/1480 train_time:151180ms step_avg:155.86ms
step:981/1480 train_time:151349ms step_avg:155.87ms
step:982/1480 train_time:151513ms step_avg:155.88ms
step:983/1480 train_time:151679ms step_avg:155.89ms
step:984/1480 train_time:151844ms step_avg:155.90ms
step:985/1480 train_time:152011ms step_avg:155.91ms
step:986/1480 train_time:152178ms step_avg:155.92ms
step:987/1480 train_time:152342ms step_avg:155.93ms
step:988/1480 train_time:152510ms step_avg:155.94ms
step:989/1480 train_time:152677ms step_avg:155.95ms
step:990/1480 train_time:152845ms step_avg:155.96ms
step:991/1480 train_time:153013ms step_avg:155.98ms
step:992/1480 train_time:153187ms step_avg:155.99ms
step:993/1480 train_time:153362ms step_avg:156.01ms
step:994/1480 train_time:153526ms step_avg:156.02ms
step:995/1480 train_time:153690ms step_avg:156.03ms
step:996/1480 train_time:153852ms step_avg:156.04ms
step:997/1480 train_time:154017ms step_avg:156.05ms
step:998/1480 train_time:154181ms step_avg:156.05ms
step:999/1480 train_time:154347ms step_avg:156.06ms
step:1000/1480 train_time:154516ms step_avg:156.08ms
step:1000/1480 val_loss:3.4398 train_time:154584ms step_avg:156.15ms
step:1001/1480 train_time:154688ms step_avg:156.09ms
step:1002/1480 train_time:154853ms step_avg:156.10ms
step:1003/1480 train_time:155025ms step_avg:156.12ms
step:1004/1480 train_time:155194ms step_avg:156.13ms
step:1005/1480 train_time:155362ms step_avg:156.14ms
step:1006/1480 train_time:155529ms step_avg:156.15ms
step:1007/1480 train_time:155695ms step_avg:156.16ms
step:1008/1480 train_time:155861ms step_avg:156.17ms
step:1009/1480 train_time:156035ms step_avg:156.19ms
step:1010/1480 train_time:156200ms step_avg:156.20ms
step:1011/1480 train_time:156365ms step_avg:156.21ms
step:1012/1480 train_time:156531ms step_avg:156.22ms
step:1013/1480 train_time:156700ms step_avg:156.23ms
step:1014/1480 train_time:156867ms step_avg:156.24ms
step:1015/1480 train_time:157037ms step_avg:156.26ms
step:1016/1480 train_time:157203ms step_avg:156.27ms
step:1017/1480 train_time:157375ms step_avg:156.28ms
step:1018/1480 train_time:157544ms step_avg:156.29ms
step:1019/1480 train_time:157713ms step_avg:156.31ms
step:1020/1480 train_time:157880ms step_avg:156.32ms
step:1021/1480 train_time:158045ms step_avg:156.33ms
step:1022/1480 train_time:158213ms step_avg:156.34ms
step:1023/1480 train_time:158380ms step_avg:156.35ms
step:1024/1480 train_time:158547ms step_avg:156.36ms
step:1025/1480 train_time:158717ms step_avg:156.37ms
step:1026/1480 train_time:158882ms step_avg:156.38ms
step:1027/1480 train_time:159050ms step_avg:156.39ms
step:1028/1480 train_time:159221ms step_avg:156.41ms
step:1029/1480 train_time:159395ms step_avg:156.42ms
step:1030/1480 train_time:159563ms step_avg:156.43ms
step:1031/1480 train_time:159728ms step_avg:156.44ms
step:1032/1480 train_time:159899ms step_avg:156.46ms
step:1033/1480 train_time:160064ms step_avg:156.47ms
step:1034/1480 train_time:160233ms step_avg:156.48ms
step:1035/1480 train_time:160401ms step_avg:156.49ms
step:1036/1480 train_time:160566ms step_avg:156.50ms
step:1037/1480 train_time:160733ms step_avg:156.51ms
step:1038/1480 train_time:160901ms step_avg:156.52ms
step:1039/1480 train_time:161072ms step_avg:156.53ms
step:1040/1480 train_time:161239ms step_avg:156.54ms
step:1041/1480 train_time:161406ms step_avg:156.55ms
step:1042/1480 train_time:161571ms step_avg:156.56ms
step:1043/1480 train_time:161736ms step_avg:156.57ms
step:1044/1480 train_time:161900ms step_avg:156.58ms
step:1045/1480 train_time:162072ms step_avg:156.59ms
step:1046/1480 train_time:162239ms step_avg:156.60ms
step:1047/1480 train_time:162406ms step_avg:156.61ms
step:1048/1480 train_time:162572ms step_avg:156.62ms
step:1049/1480 train_time:162737ms step_avg:156.63ms
step:1050/1480 train_time:162907ms step_avg:156.64ms
step:1051/1480 train_time:163078ms step_avg:156.66ms
step:1052/1480 train_time:163247ms step_avg:156.67ms
step:1053/1480 train_time:163413ms step_avg:156.68ms
step:1054/1480 train_time:163581ms step_avg:156.69ms
step:1055/1480 train_time:163746ms step_avg:156.69ms
step:1056/1480 train_time:163912ms step_avg:156.70ms
step:1057/1480 train_time:164079ms step_avg:156.71ms
step:1058/1480 train_time:164250ms step_avg:156.73ms
step:1059/1480 train_time:164423ms step_avg:156.74ms
step:1060/1480 train_time:164591ms step_avg:156.75ms
step:1061/1480 train_time:164754ms step_avg:156.76ms
step:1062/1480 train_time:164920ms step_avg:156.77ms
step:1063/1480 train_time:165086ms step_avg:156.78ms
step:1064/1480 train_time:165250ms step_avg:156.78ms
step:1065/1480 train_time:165417ms step_avg:156.79ms
step:1066/1480 train_time:165585ms step_avg:156.80ms
step:1067/1480 train_time:165754ms step_avg:156.82ms
step:1068/1480 train_time:165919ms step_avg:156.82ms
step:1069/1480 train_time:166090ms step_avg:156.84ms
step:1070/1480 train_time:166256ms step_avg:156.85ms
step:1071/1480 train_time:166430ms step_avg:156.86ms
step:1072/1480 train_time:166597ms step_avg:156.87ms
step:1073/1480 train_time:166759ms step_avg:156.88ms
step:1074/1480 train_time:166927ms step_avg:156.89ms
step:1075/1480 train_time:167098ms step_avg:156.90ms
step:1076/1480 train_time:167266ms step_avg:156.91ms
step:1077/1480 train_time:167433ms step_avg:156.92ms
step:1078/1480 train_time:167607ms step_avg:156.94ms
step:1079/1480 train_time:167779ms step_avg:156.95ms
step:1080/1480 train_time:167950ms step_avg:156.96ms
step:1081/1480 train_time:168116ms step_avg:156.97ms
step:1082/1480 train_time:168281ms step_avg:156.98ms
step:1083/1480 train_time:168448ms step_avg:156.99ms
step:1084/1480 train_time:168615ms step_avg:157.00ms
step:1085/1480 train_time:168787ms step_avg:157.01ms
step:1086/1480 train_time:168957ms step_avg:157.02ms
step:1087/1480 train_time:169124ms step_avg:157.03ms
step:1088/1480 train_time:169295ms step_avg:157.05ms
step:1089/1480 train_time:169468ms step_avg:157.06ms
step:1090/1480 train_time:169640ms step_avg:157.07ms
step:1091/1480 train_time:169807ms step_avg:157.08ms
step:1092/1480 train_time:169974ms step_avg:157.09ms
step:1093/1480 train_time:170142ms step_avg:157.10ms
step:1094/1480 train_time:170309ms step_avg:157.11ms
step:1095/1480 train_time:170474ms step_avg:157.12ms
step:1096/1480 train_time:170643ms step_avg:157.13ms
step:1097/1480 train_time:170812ms step_avg:157.14ms
step:1098/1480 train_time:170981ms step_avg:157.15ms
step:1099/1480 train_time:171152ms step_avg:157.16ms
step:1100/1480 train_time:171324ms step_avg:157.18ms
step:1101/1480 train_time:171494ms step_avg:157.19ms
step:1102/1480 train_time:171665ms step_avg:157.20ms
step:1103/1480 train_time:171841ms step_avg:157.22ms
step:1104/1480 train_time:172010ms step_avg:157.23ms
step:1105/1480 train_time:172179ms step_avg:157.24ms
step:1106/1480 train_time:172348ms step_avg:157.25ms
step:1107/1480 train_time:172517ms step_avg:157.26ms
step:1108/1480 train_time:172681ms step_avg:157.27ms
step:1109/1480 train_time:172850ms step_avg:157.28ms
step:1110/1480 train_time:173014ms step_avg:157.29ms
step:1111/1480 train_time:173180ms step_avg:157.29ms
step:1112/1480 train_time:173351ms step_avg:157.31ms
step:1113/1480 train_time:173529ms step_avg:157.32ms
step:1114/1480 train_time:173700ms step_avg:157.34ms
step:1115/1480 train_time:173873ms step_avg:157.35ms
step:1116/1480 train_time:174039ms step_avg:157.36ms
step:1117/1480 train_time:174213ms step_avg:157.37ms
step:1118/1480 train_time:174386ms step_avg:157.39ms
step:1119/1480 train_time:174552ms step_avg:157.40ms
step:1120/1480 train_time:174720ms step_avg:157.41ms
step:1121/1480 train_time:174891ms step_avg:157.42ms
step:1122/1480 train_time:175057ms step_avg:157.43ms
step:1123/1480 train_time:175225ms step_avg:157.43ms
step:1124/1480 train_time:175393ms step_avg:157.44ms
step:1125/1480 train_time:175560ms step_avg:157.45ms
step:1125/1480 val_loss:3.3860 train_time:175628ms step_avg:157.51ms
step:1126/1480 train_time:175732ms step_avg:157.47ms
step:1127/1480 train_time:175902ms step_avg:157.48ms
step:1128/1480 train_time:176075ms step_avg:157.49ms
step:1129/1480 train_time:176248ms step_avg:157.50ms
step:1130/1480 train_time:176417ms step_avg:157.52ms
step:1131/1480 train_time:176594ms step_avg:157.53ms
step:1132/1480 train_time:176761ms step_avg:157.54ms
step:1133/1480 train_time:176933ms step_avg:157.55ms
step:1134/1480 train_time:177103ms step_avg:157.56ms
step:1135/1480 train_time:177271ms step_avg:157.57ms
step:1136/1480 train_time:177441ms step_avg:157.59ms
step:1137/1480 train_time:177611ms step_avg:157.60ms
step:1138/1480 train_time:177783ms step_avg:157.61ms
step:1139/1480 train_time:177951ms step_avg:157.62ms
step:1140/1480 train_time:178119ms step_avg:157.63ms
step:1141/1480 train_time:178292ms step_avg:157.64ms
step:1142/1480 train_time:178460ms step_avg:157.65ms
step:1143/1480 train_time:178631ms step_avg:157.66ms
step:1144/1480 train_time:178801ms step_avg:157.67ms
step:1145/1480 train_time:178966ms step_avg:157.68ms
step:1146/1480 train_time:179139ms step_avg:157.69ms
step:1147/1480 train_time:179307ms step_avg:157.70ms
step:1148/1480 train_time:179478ms step_avg:157.71ms
step:1149/1480 train_time:179647ms step_avg:157.72ms
step:1150/1480 train_time:179815ms step_avg:157.73ms
step:1151/1480 train_time:179988ms step_avg:157.75ms
step:1152/1480 train_time:180160ms step_avg:157.76ms
step:1153/1480 train_time:180334ms step_avg:157.77ms
step:1154/1480 train_time:180501ms step_avg:157.78ms
step:1155/1480 train_time:180671ms step_avg:157.79ms
step:1156/1480 train_time:180847ms step_avg:157.81ms
step:1157/1480 train_time:181018ms step_avg:157.82ms
step:1158/1480 train_time:181185ms step_avg:157.83ms
step:1159/1480 train_time:181353ms step_avg:157.84ms
step:1160/1480 train_time:181519ms step_avg:157.84ms
step:1161/1480 train_time:181690ms step_avg:157.85ms
step:1162/1480 train_time:181859ms step_avg:157.86ms
step:1163/1480 train_time:182028ms step_avg:157.87ms
step:1164/1480 train_time:182197ms step_avg:157.88ms
step:1165/1480 train_time:182363ms step_avg:157.89ms
step:1166/1480 train_time:182533ms step_avg:157.90ms
step:1167/1480 train_time:182702ms step_avg:157.91ms
step:1168/1480 train_time:182870ms step_avg:157.92ms
step:1169/1480 train_time:183039ms step_avg:157.93ms
step:1170/1480 train_time:183208ms step_avg:157.94ms
step:1171/1480 train_time:183375ms step_avg:157.95ms
step:1172/1480 train_time:183542ms step_avg:157.95ms
step:1173/1480 train_time:183713ms step_avg:157.96ms
step:1174/1480 train_time:183894ms step_avg:157.98ms
step:1175/1480 train_time:184065ms step_avg:158.00ms
step:1176/1480 train_time:184238ms step_avg:158.01ms
step:1177/1480 train_time:184414ms step_avg:158.02ms
step:1178/1480 train_time:184582ms step_avg:158.03ms
step:1179/1480 train_time:184747ms step_avg:158.04ms
step:1180/1480 train_time:184926ms step_avg:158.06ms
step:1181/1480 train_time:185096ms step_avg:158.07ms
step:1182/1480 train_time:185264ms step_avg:158.07ms
step:1183/1480 train_time:185434ms step_avg:158.09ms
step:1184/1480 train_time:185601ms step_avg:158.09ms
step:1185/1480 train_time:185776ms step_avg:158.11ms
step:1186/1480 train_time:185946ms step_avg:158.12ms
step:1187/1480 train_time:186129ms step_avg:158.14ms
step:1188/1480 train_time:186296ms step_avg:158.15ms
step:1189/1480 train_time:186467ms step_avg:158.16ms
step:1190/1480 train_time:186636ms step_avg:158.17ms
step:1191/1480 train_time:186806ms step_avg:158.18ms
step:1192/1480 train_time:186972ms step_avg:158.18ms
step:1193/1480 train_time:187138ms step_avg:158.19ms
step:1194/1480 train_time:187306ms step_avg:158.20ms
step:1195/1480 train_time:187481ms step_avg:158.21ms
step:1196/1480 train_time:187662ms step_avg:158.23ms
step:1197/1480 train_time:187834ms step_avg:158.24ms
step:1198/1480 train_time:188019ms step_avg:158.26ms
step:1199/1480 train_time:188188ms step_avg:158.27ms
step:1200/1480 train_time:188358ms step_avg:158.28ms
step:1201/1480 train_time:188525ms step_avg:158.29ms
step:1202/1480 train_time:188707ms step_avg:158.31ms
step:1203/1480 train_time:188882ms step_avg:158.33ms
step:1204/1480 train_time:189057ms step_avg:158.34ms
step:1205/1480 train_time:189225ms step_avg:158.35ms
step:1206/1480 train_time:189393ms step_avg:158.36ms
step:1207/1480 train_time:189563ms step_avg:158.37ms
step:1208/1480 train_time:189730ms step_avg:158.37ms
step:1209/1480 train_time:189905ms step_avg:158.39ms
step:1210/1480 train_time:190081ms step_avg:158.40ms
step:1211/1480 train_time:190254ms step_avg:158.41ms
step:1212/1480 train_time:190425ms step_avg:158.42ms
step:1213/1480 train_time:190598ms step_avg:158.44ms
step:1214/1480 train_time:190776ms step_avg:158.45ms
step:1215/1480 train_time:190948ms step_avg:158.46ms
step:1216/1480 train_time:191118ms step_avg:158.47ms
step:1217/1480 train_time:191291ms step_avg:158.48ms
step:1218/1480 train_time:191463ms step_avg:158.50ms
step:1219/1480 train_time:191642ms step_avg:158.51ms
step:1220/1480 train_time:191812ms step_avg:158.52ms
step:1221/1480 train_time:191982ms step_avg:158.53ms
step:1222/1480 train_time:192147ms step_avg:158.54ms
step:1223/1480 train_time:192317ms step_avg:158.55ms
step:1224/1480 train_time:192496ms step_avg:158.56ms
step:1225/1480 train_time:192667ms step_avg:158.57ms
step:1226/1480 train_time:192841ms step_avg:158.59ms
step:1227/1480 train_time:193015ms step_avg:158.60ms
step:1228/1480 train_time:193184ms step_avg:158.61ms
step:1229/1480 train_time:193357ms step_avg:158.62ms
step:1230/1480 train_time:193535ms step_avg:158.64ms
step:1231/1480 train_time:193709ms step_avg:158.65ms
step:1232/1480 train_time:193884ms step_avg:158.66ms
step:1233/1480 train_time:194055ms step_avg:158.67ms
step:1234/1480 train_time:194224ms step_avg:158.68ms
step:1235/1480 train_time:194399ms step_avg:158.69ms
step:1236/1480 train_time:194567ms step_avg:158.70ms
step:1237/1480 train_time:194737ms step_avg:158.71ms
step:1238/1480 train_time:194921ms step_avg:158.73ms
step:1239/1480 train_time:195091ms step_avg:158.74ms
step:1240/1480 train_time:195262ms step_avg:158.75ms
step:1241/1480 train_time:195436ms step_avg:158.76ms
step:1242/1480 train_time:195605ms step_avg:158.77ms
step:1243/1480 train_time:195781ms step_avg:158.78ms
step:1244/1480 train_time:195946ms step_avg:158.79ms
step:1245/1480 train_time:196115ms step_avg:158.80ms
step:1246/1480 train_time:196284ms step_avg:158.81ms
step:1247/1480 train_time:196454ms step_avg:158.81ms
step:1248/1480 train_time:196622ms step_avg:158.82ms
step:1249/1480 train_time:196790ms step_avg:158.83ms
step:1250/1480 train_time:196959ms step_avg:158.84ms
step:1250/1480 val_loss:3.3351 train_time:197030ms step_avg:158.89ms
step:1251/1480 train_time:197136ms step_avg:158.85ms
step:1252/1480 train_time:197305ms step_avg:158.86ms
step:1253/1480 train_time:197473ms step_avg:158.87ms
step:1254/1480 train_time:197644ms step_avg:158.88ms
step:1255/1480 train_time:197831ms step_avg:158.90ms
step:1256/1480 train_time:198006ms step_avg:158.91ms
step:1257/1480 train_time:198175ms step_avg:158.92ms
step:1258/1480 train_time:198350ms step_avg:158.93ms
step:1259/1480 train_time:198521ms step_avg:158.94ms
step:1260/1480 train_time:198689ms step_avg:158.95ms
step:1261/1480 train_time:198859ms step_avg:158.96ms
step:1262/1480 train_time:199035ms step_avg:158.97ms
step:1263/1480 train_time:199209ms step_avg:158.99ms
step:1264/1480 train_time:199375ms step_avg:158.99ms
step:1265/1480 train_time:199542ms step_avg:159.00ms
step:1266/1480 train_time:199714ms step_avg:159.01ms
step:1267/1480 train_time:199885ms step_avg:159.02ms
step:1268/1480 train_time:200054ms step_avg:159.03ms
step:1269/1480 train_time:200230ms step_avg:159.04ms
step:1270/1480 train_time:200400ms step_avg:159.05ms
step:1271/1480 train_time:200571ms step_avg:159.06ms
step:1272/1480 train_time:200737ms step_avg:159.06ms
step:1273/1480 train_time:200908ms step_avg:159.07ms
step:1274/1480 train_time:201081ms step_avg:159.08ms
step:1275/1480 train_time:201249ms step_avg:159.09ms
step:1276/1480 train_time:201414ms step_avg:159.09ms
step:1277/1480 train_time:201585ms step_avg:159.10ms
step:1278/1480 train_time:201753ms step_avg:159.11ms
step:1279/1480 train_time:201923ms step_avg:159.12ms
step:1280/1480 train_time:202104ms step_avg:159.14ms
step:1281/1480 train_time:202273ms step_avg:159.14ms
step:1282/1480 train_time:202438ms step_avg:159.15ms
step:1283/1480 train_time:202610ms step_avg:159.16ms
step:1284/1480 train_time:202781ms step_avg:159.17ms
step:1285/1480 train_time:202951ms step_avg:159.18ms
step:1286/1480 train_time:203120ms step_avg:159.18ms
step:1287/1480 train_time:203293ms step_avg:159.20ms
step:1288/1480 train_time:203465ms step_avg:159.21ms
step:1289/1480 train_time:203651ms step_avg:159.23ms
step:1290/1480 train_time:203831ms step_avg:159.24ms
step:1291/1480 train_time:204006ms step_avg:159.26ms
step:1292/1480 train_time:204179ms step_avg:159.27ms
step:1293/1480 train_time:204354ms step_avg:159.28ms
step:1294/1480 train_time:204526ms step_avg:159.29ms
step:1295/1480 train_time:204697ms step_avg:159.30ms
step:1296/1480 train_time:204871ms step_avg:159.31ms
step:1297/1480 train_time:205042ms step_avg:159.32ms
step:1298/1480 train_time:205212ms step_avg:159.33ms
step:1299/1480 train_time:205383ms step_avg:159.34ms
step:1300/1480 train_time:205551ms step_avg:159.34ms
step:1301/1480 train_time:205719ms step_avg:159.35ms
step:1302/1480 train_time:205892ms step_avg:159.36ms
step:1303/1480 train_time:206070ms step_avg:159.37ms
step:1304/1480 train_time:206243ms step_avg:159.38ms
step:1305/1480 train_time:206412ms step_avg:159.39ms
step:1306/1480 train_time:206587ms step_avg:159.40ms
step:1307/1480 train_time:206755ms step_avg:159.41ms
step:1308/1480 train_time:206925ms step_avg:159.42ms
step:1309/1480 train_time:207096ms step_avg:159.43ms
step:1310/1480 train_time:207267ms step_avg:159.44ms
step:1311/1480 train_time:207435ms step_avg:159.44ms
step:1312/1480 train_time:207609ms step_avg:159.45ms
step:1313/1480 train_time:207777ms step_avg:159.46ms
step:1314/1480 train_time:207950ms step_avg:159.47ms
step:1315/1480 train_time:208120ms step_avg:159.48ms
step:1316/1480 train_time:208289ms step_avg:159.49ms
step:1317/1480 train_time:208459ms step_avg:159.49ms
step:1318/1480 train_time:208639ms step_avg:159.51ms
step:1319/1480 train_time:208815ms step_avg:159.52ms
step:1320/1480 train_time:208992ms step_avg:159.54ms
step:1321/1480 train_time:209165ms step_avg:159.55ms
step:1322/1480 train_time:209344ms step_avg:159.56ms
step:1323/1480 train_time:209515ms step_avg:159.57ms
step:1324/1480 train_time:209691ms step_avg:159.58ms
step:1325/1480 train_time:209874ms step_avg:159.60ms
step:1326/1480 train_time:210049ms step_avg:159.61ms
step:1327/1480 train_time:210218ms step_avg:159.62ms
step:1328/1480 train_time:210390ms step_avg:159.63ms
step:1329/1480 train_time:210586ms step_avg:159.66ms
step:1330/1480 train_time:210768ms step_avg:159.67ms
step:1331/1480 train_time:210938ms step_avg:159.68ms
step:1332/1480 train_time:211114ms step_avg:159.69ms
step:1333/1480 train_time:211288ms step_avg:159.70ms
step:1334/1480 train_time:211458ms step_avg:159.71ms
step:1335/1480 train_time:211627ms step_avg:159.72ms
step:1336/1480 train_time:211810ms step_avg:159.74ms
step:1337/1480 train_time:211986ms step_avg:159.75ms
step:1338/1480 train_time:212157ms step_avg:159.76ms
step:1339/1480 train_time:212331ms step_avg:159.77ms
step:1340/1480 train_time:212502ms step_avg:159.78ms
step:1341/1480 train_time:212670ms step_avg:159.78ms
step:1342/1480 train_time:212844ms step_avg:159.79ms
step:1343/1480 train_time:213014ms step_avg:159.80ms
step:1344/1480 train_time:213188ms step_avg:159.81ms
step:1345/1480 train_time:213367ms step_avg:159.83ms
step:1346/1480 train_time:213535ms step_avg:159.83ms
step:1347/1480 train_time:213707ms step_avg:159.84ms
step:1348/1480 train_time:213875ms step_avg:159.85ms
step:1349/1480 train_time:214044ms step_avg:159.85ms
step:1350/1480 train_time:214217ms step_avg:159.86ms
step:1351/1480 train_time:214389ms step_avg:159.87ms
step:1352/1480 train_time:214558ms step_avg:159.88ms
step:1353/1480 train_time:214734ms step_avg:159.89ms
step:1354/1480 train_time:214905ms step_avg:159.90ms
step:1355/1480 train_time:215074ms step_avg:159.91ms
step:1356/1480 train_time:215248ms step_avg:159.92ms
step:1357/1480 train_time:215421ms step_avg:159.93ms
step:1358/1480 train_time:215594ms step_avg:159.94ms
step:1359/1480 train_time:215767ms step_avg:159.95ms
step:1360/1480 train_time:215942ms step_avg:159.96ms
step:1361/1480 train_time:216119ms step_avg:159.97ms
step:1362/1480 train_time:216293ms step_avg:159.98ms
step:1363/1480 train_time:216474ms step_avg:160.00ms
step:1364/1480 train_time:216644ms step_avg:160.00ms
step:1365/1480 train_time:216811ms step_avg:160.01ms
step:1366/1480 train_time:216983ms step_avg:160.02ms
step:1367/1480 train_time:217154ms step_avg:160.02ms
step:1368/1480 train_time:217330ms step_avg:160.04ms
step:1369/1480 train_time:217512ms step_avg:160.05ms
step:1370/1480 train_time:217689ms step_avg:160.07ms
step:1371/1480 train_time:217860ms step_avg:160.07ms
step:1372/1480 train_time:218037ms step_avg:160.09ms
step:1373/1480 train_time:218207ms step_avg:160.09ms
step:1374/1480 train_time:218381ms step_avg:160.10ms
step:1375/1480 train_time:218550ms step_avg:160.11ms
step:1375/1480 val_loss:3.2964 train_time:218617ms step_avg:160.16ms
step:1376/1480 train_time:218724ms step_avg:160.12ms
step:1377/1480 train_time:218897ms step_avg:160.13ms
step:1378/1480 train_time:219065ms step_avg:160.13ms
step:1379/1480 train_time:219240ms step_avg:160.15ms
step:1380/1480 train_time:219413ms step_avg:160.16ms
step:1381/1480 train_time:219592ms step_avg:160.17ms
step:1382/1480 train_time:219763ms step_avg:160.18ms
step:1383/1480 train_time:219933ms step_avg:160.18ms
step:1384/1480 train_time:220113ms step_avg:160.20ms
step:1385/1480 train_time:220279ms step_avg:160.20ms
step:1386/1480 train_time:220450ms step_avg:160.21ms
step:1387/1480 train_time:220622ms step_avg:160.22ms
step:1388/1480 train_time:220790ms step_avg:160.22ms
step:1389/1480 train_time:220963ms step_avg:160.23ms
step:1390/1480 train_time:221131ms step_avg:160.24ms
step:1391/1480 train_time:221302ms step_avg:160.25ms
step:1392/1480 train_time:221474ms step_avg:160.26ms
step:1393/1480 train_time:221644ms step_avg:160.26ms
step:1394/1480 train_time:221814ms step_avg:160.27ms
step:1395/1480 train_time:221983ms step_avg:160.28ms
step:1396/1480 train_time:222151ms step_avg:160.28ms
step:1397/1480 train_time:222318ms step_avg:160.29ms
step:1398/1480 train_time:222486ms step_avg:160.29ms
step:1399/1480 train_time:222656ms step_avg:160.30ms
step:1400/1480 train_time:222831ms step_avg:160.31ms
step:1401/1480 train_time:222998ms step_avg:160.32ms
step:1402/1480 train_time:223168ms step_avg:160.32ms
step:1403/1480 train_time:223344ms step_avg:160.33ms
step:1404/1480 train_time:223516ms step_avg:160.34ms
step:1405/1480 train_time:223688ms step_avg:160.35ms
step:1406/1480 train_time:223863ms step_avg:160.36ms
step:1407/1480 train_time:224031ms step_avg:160.37ms
step:1408/1480 train_time:224200ms step_avg:160.37ms
step:1409/1480 train_time:224384ms step_avg:160.39ms
step:1410/1480 train_time:224554ms step_avg:160.40ms
step:1411/1480 train_time:224723ms step_avg:160.40ms
step:1412/1480 train_time:224892ms step_avg:160.41ms
step:1413/1480 train_time:225062ms step_avg:160.41ms
step:1414/1480 train_time:225234ms step_avg:160.42ms
step:1415/1480 train_time:225409ms step_avg:160.43ms
step:1416/1480 train_time:225596ms step_avg:160.45ms
step:1417/1480 train_time:225770ms step_avg:160.46ms
step:1418/1480 train_time:225942ms step_avg:160.47ms
step:1419/1480 train_time:226115ms step_avg:160.48ms
step:1420/1480 train_time:226291ms step_avg:160.49ms
step:1421/1480 train_time:226463ms step_avg:160.50ms
step:1422/1480 train_time:226635ms step_avg:160.51ms
step:1423/1480 train_time:226805ms step_avg:160.51ms
step:1424/1480 train_time:226983ms step_avg:160.53ms
step:1425/1480 train_time:227164ms step_avg:160.54ms
step:1426/1480 train_time:227336ms step_avg:160.55ms
step:1427/1480 train_time:227511ms step_avg:160.56ms
step:1428/1480 train_time:227684ms step_avg:160.57ms
step:1429/1480 train_time:227853ms step_avg:160.57ms
step:1430/1480 train_time:228028ms step_avg:160.58ms
step:1431/1480 train_time:228205ms step_avg:160.59ms
step:1432/1480 train_time:228383ms step_avg:160.61ms
step:1433/1480 train_time:228562ms step_avg:160.62ms
step:1434/1480 train_time:228742ms step_avg:160.63ms
step:1435/1480 train_time:228917ms step_avg:160.64ms
step:1436/1480 train_time:229090ms step_avg:160.65ms
step:1437/1480 train_time:229259ms step_avg:160.66ms
step:1438/1480 train_time:229427ms step_avg:160.66ms
step:1439/1480 train_time:229601ms step_avg:160.67ms
step:1440/1480 train_time:229768ms step_avg:160.68ms
step:1441/1480 train_time:229939ms step_avg:160.68ms
step:1442/1480 train_time:230115ms step_avg:160.70ms
step:1443/1480 train_time:230305ms step_avg:160.72ms
step:1444/1480 train_time:230477ms step_avg:160.72ms
step:1445/1480 train_time:230649ms step_avg:160.73ms
step:1446/1480 train_time:230825ms step_avg:160.74ms
step:1447/1480 train_time:231002ms step_avg:160.75ms
step:1448/1480 train_time:231173ms step_avg:160.76ms
step:1449/1480 train_time:231347ms step_avg:160.77ms
step:1450/1480 train_time:231521ms step_avg:160.78ms
step:1451/1480 train_time:231690ms step_avg:160.78ms
step:1452/1480 train_time:231862ms step_avg:160.79ms
step:1453/1480 train_time:232031ms step_avg:160.80ms
step:1454/1480 train_time:232203ms step_avg:160.81ms
step:1455/1480 train_time:232382ms step_avg:160.82ms
step:1456/1480 train_time:232556ms step_avg:160.83ms
step:1457/1480 train_time:232726ms step_avg:160.83ms
step:1458/1480 train_time:232898ms step_avg:160.84ms
step:1459/1480 train_time:233074ms step_avg:160.85ms
step:1460/1480 train_time:233245ms step_avg:160.86ms
step:1461/1480 train_time:233421ms step_avg:160.87ms
step:1462/1480 train_time:233590ms step_avg:160.87ms
step:1463/1480 train_time:233766ms step_avg:160.89ms
step:1464/1480 train_time:233942ms step_avg:160.90ms
step:1465/1480 train_time:234112ms step_avg:160.90ms
step:1466/1480 train_time:234284ms step_avg:160.91ms
step:1467/1480 train_time:234457ms step_avg:160.92ms
step:1468/1480 train_time:234626ms step_avg:160.92ms
step:1469/1480 train_time:234800ms step_avg:160.93ms
step:1470/1480 train_time:234980ms step_avg:160.95ms
step:1471/1480 train_time:235166ms step_avg:160.96ms
step:1472/1480 train_time:235347ms step_avg:160.98ms
step:1473/1480 train_time:235518ms step_avg:160.98ms
step:1474/1480 train_time:235697ms step_avg:160.99ms
step:1475/1480 train_time:235875ms step_avg:161.01ms
step:1476/1480 train_time:236048ms step_avg:161.01ms
step:1477/1480 train_time:236228ms step_avg:161.03ms
step:1478/1480 train_time:236411ms step_avg:161.04ms
step:1479/1480 train_time:236585ms step_avg:161.05ms
step:1480/1480 train_time:236758ms step_avg:161.06ms
step:1480/1480 val_loss:3.2778 train_time:236828ms step_avg:161.11ms