be800e9a-90a5-457d-8f86-a943f4dea20c.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
from dataclasses import dataclass
from pathlib import Path

import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
# Use of FlexAttention contributed by @KoszarskyB
from torch.nn.attention.flex_attention import BlockMask, flex_attention

# -----------------------------------------------------------------------------
# Muon optimizer

def zeropower_via_svd(G, steps=None):
    U, S, V = G.svd()
    return U @ V.T

@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert len(G.shape) == 2
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    X /= (X.norm() + eps) # ensure top singular value <= 1
    if G.size(0) > G.size(1):
        X = X.T
    for _ in range(steps):
        A = X @ X.T
        B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    if G.size(0) > G.size(1):
        X = X.T
    return X

zeropower_backends = dict(svd=zeropower_via_svd, newtonschulz5=zeropower_via_newtonschulz5)

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven't tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        backend: The chosen backend for the orthogonalization step. (recommended: 'newtonschulz5')
        backend_steps: The number of iteration steps to use in the backend, if it is iterative.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True,
                 backend='newtonschulz5', backend_steps=5):
        self.num_process = int(os.environ['WORLD_SIZE'])
        self.rank = int(os.environ["RANK"])
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, backend=backend, backend_steps=backend_steps)
        params: "list[torch.Tensor]" = list(params)
        assert all(isinstance(p, torch.Tensor) for p in params)
        sizes = {p.numel() for p in params}
        param_groups = [
            {
                "params": [p for p in params if p.numel() == size],
                "update_buffer": [
                    torch.empty(size, device="cuda", dtype=torch.bfloat16)
                    for _ in range(self.num_process)
                ],
            }
            for size in sizes
        ]
        super().__init__(param_groups, defaults)

    def step(self):
        for group in self.param_groups:
            lr: float = group["lr"]
            momentum: float = group["momentum"]
            nesterov: bool = group["nesterov"]
            zeropower_backend = zeropower_backends[group["backend"]]
            backend_steps: int = group["backend_steps"]
            update_buffers: "list[torch.Tensor]" = group["update_buffer"]
            # generate weight updates in distributed fashion
            params: "list[torch.Tensor]" = group["params"]
            assert len(params) % self.num_process == 0
            handle = None
            params_world = None
            def update_prev():
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffers):
                    p_world.data.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.num_process]:
                p = params[base_i + self.rank]
                g = p.grad
                assert g is not None
                state = self.state[p] 
                if "momentum_buffer" not in state:
                    state["momentum_buffer"] = torch.zeros_like(g)
                buf: torch.Tensor = state["momentum_buffer"]
                buf.lerp_(g, 1 - momentum)
                g = g.lerp_(buf, momentum) if nesterov else buf
                g = zeropower_backend(g, steps=backend_steps).flatten()
                update_prev()
                handle = dist.all_gather(update_buffers, g, async_op=True)
                params_world = params[base_i : base_i + self.num_process]
            update_prev()


# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):

    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=False)

    def forward(self, x):
        return F.linear(x, self.weight.to(x.dtype))

class Rotary(torch.nn.Module):

    def __init__(self, dim, base=10000):
        super().__init__()
        self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None

    def forward(self, x):
        seq_len = x.shape[1]
        if seq_len != self.seq_len_cached:
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.outer(t, self.inv_freq)
            self.seq_len_cached = seq_len
            self.cos_cached = freqs.cos()
            self.sin_cached = freqs.sin()
        cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
        # apply_rotary_emb(x, cos, sin)
        x1, x2 = x.chunk(2, dim=3)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x)

class CausalSelfAttention(nn.Module):

    def __init__(self, dim, n_head):
        super().__init__()
        assert dim % n_head == 0
        self.n_head = n_head
        self.c_q = CastedLinear(dim, dim)
        self.c_k = CastedLinear(dim, dim)
        self.c_v = CastedLinear(dim, dim)
        # value residual lambda
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5])) # @Grad62304977
        # rotary embeddings
        self.rotary = Rotary(dim // n_head) # dim // n_head = head_dim
        # output projection
        self.c_proj = CastedLinear(dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor, vi: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q: torch.Tensor = self.c_q(x).view(B, T, self.n_head, -1)
        k: torch.Tensor = self.c_k(x).view(B, T, self.n_head, -1)
        v: torch.Tensor = self.c_v(x).view(B, T, self.n_head, -1)
        v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @Grad62304977
        q, k = norm(q), norm(k) # QK norm suggested by @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask)
        y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):

    def __init__(self, dim: int):
        super().__init__()
        self.c_fc   = CastedLinear(dim, 4 * dim)
        self.c_proj = CastedLinear(4 * dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):

    def __init__(self, config):
        super().__init__()
        self.attn = CausalSelfAttention(config.n_embd, config.n_head)
        self.mlp = MLP(config.n_embd)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x: torch.Tensor, vi: torch.Tensor, x0: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        x = x + self.attn(norm(x), vi, block_mask)
        x = x + self.mlp(norm(x))
        return x

# -----------------------------------------------------------------------------
# The main GPT-2 model

@dataclass
class GPTConfig:
    vocab_size : int = 50304
    n_layer : int = 12
    n_head : int = 6 # head dim 128 suggested by @Grad62304977
    n_embd : int = 768
    lm_head_softcap : int = 30

class GPT(nn.Module):

    def __init__(self, config: GPTConfig):
        super().__init__()
        self.n_layer = config.n_layer
        self.lm_head_softcap = config.lm_head_softcap

        # U-net design by @brendanh0gan
        self.num_encoder_layers = config.n_layer // 2 # Half of the layers for encoder
        self.num_decoder_layers = config.n_layer - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))

        self.transformer = nn.ModuleDict(dict(
            wte = nn.Embedding(config.vocab_size, config.n_embd),
            # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
            # U-net structure on token value embeddings by @leloykun
            vte = nn.Embedding(config.vocab_size, config.n_embd*self.num_encoder_layers),
            h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
        ))
        self.lm_head = CastedLinear(config.n_embd, config.vocab_size)
        self.lm_head.weight.data.zero_() # @Grad62304977

    def forward(self, idx: torch.Tensor, target: torch.Tensor, sliding_window: torch.Tensor) -> torch.Tensor:
        BLOCK_SIZE = 128
        assert idx.ndim == 1
        docs = (idx == 50256).cumsum(0)
        docs_low = docs.reshape(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.reshape(-1, BLOCK_SIZE)[:, -1].contiguous()
        def document_sliding_window_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            window_mask = q_idx - kv_idx < sliding_window
            return causal_mask & document_mask & window_mask

        S = len(idx)
        def create_sliding_window_causal_mask(S: int, sliding_window: torch.Tensor):
            kv_idx = block_idx = torch.arange(S // BLOCK_SIZE, dtype=torch.int32, device="cuda")
            q_idx = block_idx[:, None]
            causal_mask = q_idx >= kv_idx
            document_mask = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
            window_mask = q_idx - kv_idx < ((sliding_window + BLOCK_SIZE - 1) // BLOCK_SIZE)
            dense_mask = causal_mask & document_mask & window_mask
            dense_mask = dense_mask.to(torch.int32)
            num_blocks = dense_mask.sum(dim=-1).to(torch.int32)
            indices = torch.argsort(dense_mask, dim=-1, descending=True, stable=True).to(torch.int32)
            num_blocks = num_blocks[None, None, :].contiguous()
            indices = indices[None, None, :].contiguous()
            return BlockMask.from_kv_blocks(num_blocks, indices, BLOCK_SIZE=BLOCK_SIZE, mask_mod=document_sliding_window_causal)
        block_mask = create_sliding_window_causal_mask(S, sliding_window)

        # forward the GPT model itself
        x = self.transformer.wte(idx[None]) # token embeddings of shape (b, t, n_embd)
        x = norm(x) # @Grad62304977
        x0 = x
        vi = self.transformer.vte(idx[None]).chunk(self.num_encoder_layers, dim=-1)

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        for i in range(self.num_encoder_layers):
            x = self.transformer.h[i](x, vi[i], x0, block_mask)
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            # U-net structure on token value embeddings by @leloykun
            x = self.transformer.h[self.num_encoder_layers + i](x, vi[self.num_encoder_layers-1-i], x0, block_mask)

        x = norm(x)
        logits = self.lm_head(x)
        logits = self.lm_head_softcap * torch.tanh(logits / self.lm_head_softcap) # @Grad62304977
        logits = logits.float()
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), target.view(-1))
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _peek_data_shard(file: Path):
    # only reads the header, returns header data
    # header is 256 int32
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    return int(header[2]) # number of tokens (claimed)

def _load_data_shard(file: Path, ntok: int):
    with file.open("rb") as f:
        tokens = torch.empty(ntok, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy())
        assert nbytes == 2 * ntok, "number of tokens read does not match header?"
    return tokens

class DistributedDataLoader:
    def __init__(self, filename_pattern, T, process_rank, num_processes):
        self.process_rank = process_rank
        self.num_processes = num_processes
        self.T = T

        # glob files that match the pattern
        self.files = sorted(Path.cwd().glob(filename_pattern))
        assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"

        # load and validate all data shards, count number of tokens in total
        self.ntoks = [_peek_data_shard(file) for file in self.files]
        assert min(self.ntoks) >= num_processes * T + 1
        self.ntok_total = sum(self.ntoks)

        self.reset()

    def reset(self):
        self.current_shard = -1
        self.advance()

    def advance(self): # advance to next data shard
        self.current_shard = (self.current_shard + 1) % len(self.files)
        self.current_position = self.process_rank * self.T
        self.tokens = _load_data_shard(self.files[self.current_shard], self.ntoks[self.current_shard])

    def next_batch(self):
        batch_size = self.T * self.num_processes
        buf = self.tokens[self.current_position:self.current_position+self.T+1]
        # host side async is sufficient;
        # no performance improvement was observed when introducing a separate stream.
        x = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
        y = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
        # advance current position and load next shard if necessary
        self.current_position += batch_size
        if self.current_position + batch_size + 1 >= len(self.tokens):
            self.advance()
        return x, y

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data hyperparams
    input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
    input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
    # optimization hyperparams
    batch_size : int = 8 # batch size, in sequences, across all devices
    sequence_length : int = 64*1024 # sequence length, in tokens
    num_iterations : int = 1480 # number of iterations to run
    warmup_iters : int = 0
    cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
    weight_decay : float = 0
    # evaluation and logging hyperparams
    val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
    val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()

# set up DDP (distributed data parallel). torchrun sets this env variable
assert torch.cuda.is_available()
dist.init_process_group(backend='nccl')
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
device = f'cuda:{ddp_local_rank}'
torch.cuda.set_device(device)
print(f"using device: {device}")
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = str(uuid.uuid4())
    logdir = 'logs/%s/' % run_id
    # os.makedirs(logdir, exist_ok=True)
    logfile = 'logs/%s.txt' % run_id
    # create the log file
    with open(logfile, "w") as f:
        # begin the log by printing this file (the Python code)
        f.write(code)
        f.write('='*100 + '\n')
def print0(s, logonly=False):
    if master_process:
        with open(logfile, "a") as f:
            if not logonly:
                print(s)
            f.write(s+'\n')
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)

# convenience variables
T = args.sequence_length
# calculate the number of steps to take in the val loop.
assert args.val_tokens % (T * ddp_world_size) == 0
val_steps = args.val_tokens // (T * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size
assert train_accumulation_steps == 1

# load tokens
train_loader = DistributedDataLoader(args.input_bin, T, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, T, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.ntok_total} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.ntok_total} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
x, y = train_loader.next_batch()

# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, n_layer=12, n_head=6, n_embd=768))
model = model.cuda().bfloat16()
for m in model.modules():
    if isinstance(m, CastedLinear):
        m.float()
if hasattr(config, "coordinate_descent_tuning"):
    config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank])
raw_model = model.module # always contains the "raw" unwrapped model

# init the optimizer(s)
optimizer1 = torch.optim.Adam([raw_model.transformer.wte.weight, raw_model.transformer.vte.weight], lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.transformer.h.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
    assert it <= args.num_iterations
    # 1) linear warmup for warmup_iters steps
    if it < args.warmup_iters:
        return (it+1) / args.warmup_iters
    # 2) constant lr for a while
    elif it < args.num_iterations - args.cooldown_iters:
        return 1.0
    # 3) linear cooldown
    else:
        decay_ratio = (args.num_iterations - it) / args.cooldown_iters
        return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]

sliding_window_size = torch.tensor(64, dtype=torch.int32, device="cuda")
sw_size_prev = 64
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
    last_step = (step == args.num_iterations)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Set the sliding window size for the current step, in chunks of 64. By @fernbear.bsky.social
    sw_size =  64 * int((64 + (1792 - 64) * step / args.num_iterations) // 64)
    if sw_size != sw_size_prev:
        sliding_window_size.copy_(sw_size, non_blocking=True)
        sw_size_prev = sw_size

    # once in a while evaluate the validation dataset
    if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # run validation batches
        model.eval()
        val_loader.reset()
        val_loss = 0.0
        for _ in range(val_steps):
            with torch.no_grad():
                x_val, y_val = val_loader.next_batch()
                val_loss += model(x_val, y_val, sliding_window=sliding_window_size)
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        val_loss /= val_steps
        # log val loss to console and to logfile
        print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # save the state of the training process
        log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
        # torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    # bit confusing: we want to make sure to eval on 0th iteration
    # but also after the very last iteration. so we loop for step <= num_iterations
    # instead of just < num_iterations (one extra due to <=), only to do
    # the validation/sampling one last time, and then we break right here as we're done.
    if last_step:
        break

    # --------------- TRAINING SECTION BEGIN -----------------
    model.train()
    loss = model(x, y, sliding_window=sliding_window_size)
    loss.backward()
    del loss
    # advance the dataset for the next batch
    x, y = train_loader.next_batch()
    # momentum warmup for Muon
    frac = min(step/300, 1)
    for group in optimizer3.param_groups:
        group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # --------------- TRAINING SECTION END -------------------
    # everything that follows now is just diagnostics, prints, logging, etc.
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")

if master_process:
    print(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")

# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()
====================================================================================================
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Sun Dec  8 11:52:11 2024       
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.6     |
|-----------------------------------------+----------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
|                                         |                      |               MIG M. |
|=========================================+======================+======================|
|   0  NVIDIA H100 80GB HBM3          On  | 00000000:65:02.0 Off |                    0 |
| N/A   36C    P0              74W / 700W |      7MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  | 00000000:67:02.0 Off |                    0 |
| N/A   46C    P0             130W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  | 00000000:69:02.0 Off |                    0 |
| N/A   46C    P0             124W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  | 00000000:6B:02.0 Off |                    0 |
| N/A   39C    P0             102W / 700W |     27MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  | 00000000:6F:02.0 Off |                    0 |
| N/A   39C    P0             117W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  | 00000000:71:02.0 Off |                    0 |
| N/A   45C    P0             113W / 700W |    533MiB / 81559MiB |      2%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  | 00000000:73:02.0 Off |                    0 |
| N/A   46C    P0             127W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  | 00000000:75:02.0 Off |                    0 |
| N/A   38C    P0             124W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
                                                                                         
+---------------------------------------------------------------------------------------+
| Processes:                                                                            |
|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
|        ID   ID                                                             Usage      |
|=======================================================================================|
+---------------------------------------------------------------------------------------+

====================================================================================================
Training DataLoader: total number of tokens: 3200000000 across 32 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:23251ms step_avg:nanms
step:2/1480 train_time:23337ms step_avg:nanms
step:3/1480 train_time:23475ms step_avg:nanms
step:4/1480 train_time:23615ms step_avg:nanms
step:5/1480 train_time:23755ms step_avg:nanms
step:6/1480 train_time:23896ms step_avg:nanms
step:7/1480 train_time:24036ms step_avg:nanms
step:8/1480 train_time:24178ms step_avg:nanms
step:9/1480 train_time:24327ms step_avg:nanms
step:10/1480 train_time:24471ms step_avg:nanms
step:11/1480 train_time:141ms step_avg:nanms
step:12/1480 train_time:282ms step_avg:nanms
step:13/1480 train_time:422ms step_avg:140.82ms
step:14/1480 train_time:564ms step_avg:141.08ms
step:15/1480 train_time:704ms step_avg:140.79ms
step:16/1480 train_time:848ms step_avg:141.38ms
step:17/1480 train_time:993ms step_avg:141.86ms
step:18/1480 train_time:1137ms step_avg:142.10ms
step:19/1480 train_time:1279ms step_avg:142.12ms
step:20/1480 train_time:1420ms step_avg:142.05ms
step:21/1480 train_time:1562ms step_avg:141.98ms
step:22/1480 train_time:1703ms step_avg:141.89ms
step:23/1480 train_time:1846ms step_avg:141.97ms
step:24/1480 train_time:1991ms step_avg:142.19ms
step:25/1480 train_time:2135ms step_avg:142.34ms
step:26/1480 train_time:2278ms step_avg:142.40ms
step:27/1480 train_time:2420ms step_avg:142.33ms
step:28/1480 train_time:2561ms step_avg:142.26ms
step:29/1480 train_time:2702ms step_avg:142.22ms
step:30/1480 train_time:2846ms step_avg:142.29ms
step:31/1480 train_time:2994ms step_avg:142.56ms
step:32/1480 train_time:3135ms step_avg:142.49ms
step:33/1480 train_time:3278ms step_avg:142.54ms
step:34/1480 train_time:3420ms step_avg:142.49ms
step:35/1480 train_time:3562ms step_avg:142.47ms
step:36/1480 train_time:3702ms step_avg:142.39ms
step:37/1480 train_time:3843ms step_avg:142.35ms
step:38/1480 train_time:3988ms step_avg:142.42ms
step:39/1480 train_time:4132ms step_avg:142.48ms
step:40/1480 train_time:4276ms step_avg:142.52ms
step:41/1480 train_time:4419ms step_avg:142.55ms
step:42/1480 train_time:4562ms step_avg:142.55ms
step:43/1480 train_time:4703ms step_avg:142.52ms
step:44/1480 train_time:4844ms step_avg:142.48ms
step:45/1480 train_time:4986ms step_avg:142.47ms
step:46/1480 train_time:5131ms step_avg:142.54ms
step:47/1480 train_time:5276ms step_avg:142.60ms
step:48/1480 train_time:5418ms step_avg:142.58ms
step:49/1480 train_time:5561ms step_avg:142.58ms
step:50/1480 train_time:5703ms step_avg:142.56ms
step:51/1480 train_time:5844ms step_avg:142.53ms
step:52/1480 train_time:5987ms step_avg:142.55ms
step:53/1480 train_time:6132ms step_avg:142.60ms
step:54/1480 train_time:6276ms step_avg:142.64ms
step:55/1480 train_time:6419ms step_avg:142.63ms
step:56/1480 train_time:6562ms step_avg:142.65ms
step:57/1480 train_time:6703ms step_avg:142.61ms
step:58/1480 train_time:6847ms step_avg:142.64ms
step:59/1480 train_time:6991ms step_avg:142.67ms
step:60/1480 train_time:7136ms step_avg:142.72ms
step:61/1480 train_time:7280ms step_avg:142.74ms
step:62/1480 train_time:7422ms step_avg:142.74ms
step:63/1480 train_time:7564ms step_avg:142.71ms
step:64/1480 train_time:7704ms step_avg:142.67ms
step:65/1480 train_time:7848ms step_avg:142.70ms
step:66/1480 train_time:7993ms step_avg:142.73ms
step:67/1480 train_time:8137ms step_avg:142.75ms
step:68/1480 train_time:8279ms step_avg:142.74ms
step:69/1480 train_time:8421ms step_avg:142.72ms
step:70/1480 train_time:8562ms step_avg:142.70ms
step:71/1480 train_time:8702ms step_avg:142.66ms
step:72/1480 train_time:8847ms step_avg:142.70ms
step:73/1480 train_time:8991ms step_avg:142.72ms
step:74/1480 train_time:9137ms step_avg:142.77ms
step:75/1480 train_time:9280ms step_avg:142.77ms
step:76/1480 train_time:9422ms step_avg:142.75ms
step:77/1480 train_time:9563ms step_avg:142.74ms
step:78/1480 train_time:9704ms step_avg:142.70ms
step:79/1480 train_time:9844ms step_avg:142.67ms
step:80/1480 train_time:9986ms step_avg:142.66ms
step:81/1480 train_time:10134ms step_avg:142.73ms
step:82/1480 train_time:10278ms step_avg:142.75ms
step:83/1480 train_time:10421ms step_avg:142.75ms
step:84/1480 train_time:10564ms step_avg:142.75ms
step:85/1480 train_time:10704ms step_avg:142.72ms
step:86/1480 train_time:10845ms step_avg:142.70ms
step:87/1480 train_time:10987ms step_avg:142.69ms
step:88/1480 train_time:11131ms step_avg:142.70ms
step:89/1480 train_time:11275ms step_avg:142.72ms
step:90/1480 train_time:11418ms step_avg:142.73ms
step:91/1480 train_time:11561ms step_avg:142.73ms
step:92/1480 train_time:11702ms step_avg:142.71ms
step:93/1480 train_time:11842ms step_avg:142.68ms
step:94/1480 train_time:11986ms step_avg:142.69ms
step:95/1480 train_time:12130ms step_avg:142.70ms
step:96/1480 train_time:12273ms step_avg:142.71ms
step:97/1480 train_time:12416ms step_avg:142.71ms
step:98/1480 train_time:12559ms step_avg:142.71ms
step:99/1480 train_time:12701ms step_avg:142.70ms
step:100/1480 train_time:12843ms step_avg:142.70ms
step:101/1480 train_time:12986ms step_avg:142.70ms
step:102/1480 train_time:13130ms step_avg:142.72ms
step:103/1480 train_time:13275ms step_avg:142.74ms
step:104/1480 train_time:13418ms step_avg:142.74ms
step:105/1480 train_time:13560ms step_avg:142.74ms
step:106/1480 train_time:13701ms step_avg:142.72ms
step:107/1480 train_time:13842ms step_avg:142.70ms
step:108/1480 train_time:13985ms step_avg:142.70ms
step:109/1480 train_time:14129ms step_avg:142.72ms
step:110/1480 train_time:14274ms step_avg:142.74ms
step:111/1480 train_time:14419ms step_avg:142.77ms
step:112/1480 train_time:14566ms step_avg:142.80ms
step:113/1480 train_time:14712ms step_avg:142.84ms
step:114/1480 train_time:14859ms step_avg:142.88ms
step:115/1480 train_time:15006ms step_avg:142.91ms
step:116/1480 train_time:15154ms step_avg:142.96ms
step:117/1480 train_time:15302ms step_avg:143.01ms
step:118/1480 train_time:15448ms step_avg:143.04ms
step:119/1480 train_time:15596ms step_avg:143.09ms
step:120/1480 train_time:15743ms step_avg:143.12ms
step:121/1480 train_time:15889ms step_avg:143.15ms
step:122/1480 train_time:16036ms step_avg:143.18ms
step:123/1480 train_time:16184ms step_avg:143.22ms
step:124/1480 train_time:16332ms step_avg:143.26ms
step:125/1480 train_time:16480ms step_avg:143.31ms
step:125/1480 val_loss:4.4268 train_time:16537ms step_avg:143.80ms
step:126/1480 train_time:16633ms step_avg:143.39ms
step:127/1480 train_time:16781ms step_avg:143.43ms
step:128/1480 train_time:16927ms step_avg:143.45ms
step:129/1480 train_time:17073ms step_avg:143.47ms
step:130/1480 train_time:17220ms step_avg:143.50ms
step:131/1480 train_time:17364ms step_avg:143.51ms
step:132/1480 train_time:17511ms step_avg:143.53ms
step:133/1480 train_time:17659ms step_avg:143.57ms
step:134/1480 train_time:17808ms step_avg:143.61ms
step:135/1480 train_time:17955ms step_avg:143.64ms
step:136/1480 train_time:18101ms step_avg:143.65ms
step:137/1480 train_time:18246ms step_avg:143.67ms
step:138/1480 train_time:18393ms step_avg:143.69ms
step:139/1480 train_time:18539ms step_avg:143.72ms
step:140/1480 train_time:18687ms step_avg:143.74ms
step:141/1480 train_time:18835ms step_avg:143.78ms
step:142/1480 train_time:18981ms step_avg:143.80ms
step:143/1480 train_time:19130ms step_avg:143.83ms
step:144/1480 train_time:19275ms step_avg:143.85ms
step:145/1480 train_time:19420ms step_avg:143.86ms
step:146/1480 train_time:19568ms step_avg:143.88ms
step:147/1480 train_time:19716ms step_avg:143.91ms
step:148/1480 train_time:19862ms step_avg:143.93ms
step:149/1480 train_time:20010ms step_avg:143.96ms
step:150/1480 train_time:20157ms step_avg:143.98ms
step:151/1480 train_time:20303ms step_avg:143.99ms
step:152/1480 train_time:20449ms step_avg:144.01ms
step:153/1480 train_time:20597ms step_avg:144.03ms
step:154/1480 train_time:20741ms step_avg:144.04ms
step:155/1480 train_time:20889ms step_avg:144.06ms
step:156/1480 train_time:21036ms step_avg:144.08ms
step:157/1480 train_time:21181ms step_avg:144.09ms
step:158/1480 train_time:21328ms step_avg:144.11ms
step:159/1480 train_time:21475ms step_avg:144.13ms
step:160/1480 train_time:21621ms step_avg:144.14ms
step:161/1480 train_time:21769ms step_avg:144.16ms
step:162/1480 train_time:21915ms step_avg:144.18ms
step:163/1480 train_time:22060ms step_avg:144.18ms
step:164/1480 train_time:22207ms step_avg:144.20ms
step:165/1480 train_time:22355ms step_avg:144.22ms
step:166/1480 train_time:22500ms step_avg:144.23ms
step:167/1480 train_time:22647ms step_avg:144.25ms
step:168/1480 train_time:22795ms step_avg:144.27ms
step:169/1480 train_time:22941ms step_avg:144.28ms
step:170/1480 train_time:23088ms step_avg:144.30ms
step:171/1480 train_time:23236ms step_avg:144.32ms
step:172/1480 train_time:23382ms step_avg:144.33ms
step:173/1480 train_time:23529ms step_avg:144.35ms
step:174/1480 train_time:23676ms step_avg:144.37ms
step:175/1480 train_time:23821ms step_avg:144.37ms
step:176/1480 train_time:23970ms step_avg:144.39ms
step:177/1480 train_time:24117ms step_avg:144.41ms
step:178/1480 train_time:24262ms step_avg:144.42ms
step:179/1480 train_time:24410ms step_avg:144.44ms
step:180/1480 train_time:24557ms step_avg:144.45ms
step:181/1480 train_time:24703ms step_avg:144.46ms
step:182/1480 train_time:24850ms step_avg:144.48ms
step:183/1480 train_time:24998ms step_avg:144.50ms
step:184/1480 train_time:25144ms step_avg:144.51ms
step:185/1480 train_time:25292ms step_avg:144.53ms
step:186/1480 train_time:25438ms step_avg:144.54ms
step:187/1480 train_time:25585ms step_avg:144.55ms
step:188/1480 train_time:25732ms step_avg:144.56ms
step:189/1480 train_time:25879ms step_avg:144.57ms
step:190/1480 train_time:26026ms step_avg:144.59ms
step:191/1480 train_time:26173ms step_avg:144.60ms
step:192/1480 train_time:26320ms step_avg:144.61ms
step:193/1480 train_time:26467ms step_avg:144.63ms
step:194/1480 train_time:26613ms step_avg:144.64ms
step:195/1480 train_time:26759ms step_avg:144.65ms
step:196/1480 train_time:26906ms step_avg:144.66ms
step:197/1480 train_time:27053ms step_avg:144.67ms
step:198/1480 train_time:27199ms step_avg:144.68ms
step:199/1480 train_time:27346ms step_avg:144.69ms
step:200/1480 train_time:27494ms step_avg:144.71ms
step:201/1480 train_time:27639ms step_avg:144.71ms
step:202/1480 train_time:27788ms step_avg:144.73ms
step:203/1480 train_time:27935ms step_avg:144.74ms
step:204/1480 train_time:28081ms step_avg:144.75ms
step:205/1480 train_time:28229ms step_avg:144.77ms
step:206/1480 train_time:28376ms step_avg:144.77ms
step:207/1480 train_time:28523ms step_avg:144.78ms
step:208/1480 train_time:28670ms step_avg:144.80ms
step:209/1480 train_time:28817ms step_avg:144.81ms
step:210/1480 train_time:28962ms step_avg:144.81ms
step:211/1480 train_time:29109ms step_avg:144.82ms
step:212/1480 train_time:29257ms step_avg:144.83ms
step:213/1480 train_time:29402ms step_avg:144.84ms
step:214/1480 train_time:29549ms step_avg:144.85ms
step:215/1480 train_time:29697ms step_avg:144.86ms
step:216/1480 train_time:29842ms step_avg:144.87ms
step:217/1480 train_time:29990ms step_avg:144.88ms
step:218/1480 train_time:30136ms step_avg:144.89ms
step:219/1480 train_time:30282ms step_avg:144.89ms
step:220/1480 train_time:30431ms step_avg:144.91ms
step:221/1480 train_time:30579ms step_avg:144.92ms
step:222/1480 train_time:30730ms step_avg:144.95ms
step:223/1480 train_time:30880ms step_avg:144.98ms
step:224/1480 train_time:31031ms step_avg:145.01ms
step:225/1480 train_time:31181ms step_avg:145.03ms
step:226/1480 train_time:31332ms step_avg:145.06ms
step:227/1480 train_time:31482ms step_avg:145.08ms
step:228/1480 train_time:31631ms step_avg:145.10ms
step:229/1480 train_time:31781ms step_avg:145.12ms
step:230/1480 train_time:31932ms step_avg:145.14ms
step:231/1480 train_time:32081ms step_avg:145.16ms
step:232/1480 train_time:32233ms step_avg:145.20ms
step:233/1480 train_time:32384ms step_avg:145.22ms
step:234/1480 train_time:32535ms step_avg:145.25ms
step:235/1480 train_time:32686ms step_avg:145.27ms
step:236/1480 train_time:32836ms step_avg:145.29ms
step:237/1480 train_time:32985ms step_avg:145.31ms
step:238/1480 train_time:33136ms step_avg:145.33ms
step:239/1480 train_time:33286ms step_avg:145.36ms
step:240/1480 train_time:33437ms step_avg:145.38ms
step:241/1480 train_time:33585ms step_avg:145.39ms
step:242/1480 train_time:33735ms step_avg:145.41ms
step:243/1480 train_time:33885ms step_avg:145.43ms
step:244/1480 train_time:34036ms step_avg:145.45ms
step:245/1480 train_time:34186ms step_avg:145.47ms
step:246/1480 train_time:34337ms step_avg:145.50ms
step:247/1480 train_time:34489ms step_avg:145.52ms
step:248/1480 train_time:34639ms step_avg:145.54ms
step:249/1480 train_time:34789ms step_avg:145.56ms
step:250/1480 train_time:34939ms step_avg:145.58ms
step:250/1480 val_loss:3.9953 train_time:34998ms step_avg:145.82ms
step:251/1480 train_time:35095ms step_avg:145.62ms
step:252/1480 train_time:35247ms step_avg:145.65ms
step:253/1480 train_time:35397ms step_avg:145.67ms
step:254/1480 train_time:35546ms step_avg:145.68ms
step:255/1480 train_time:35694ms step_avg:145.69ms
step:256/1480 train_time:35844ms step_avg:145.71ms
step:257/1480 train_time:35994ms step_avg:145.72ms
step:258/1480 train_time:36147ms step_avg:145.76ms
step:259/1480 train_time:36299ms step_avg:145.78ms
step:260/1480 train_time:36452ms step_avg:145.81ms
step:261/1480 train_time:36602ms step_avg:145.83ms
step:262/1480 train_time:36752ms step_avg:145.84ms
step:263/1480 train_time:36901ms step_avg:145.85ms
step:264/1480 train_time:37053ms step_avg:145.88ms
step:265/1480 train_time:37204ms step_avg:145.90ms
step:266/1480 train_time:37355ms step_avg:145.92ms
step:267/1480 train_time:37505ms step_avg:145.93ms
step:268/1480 train_time:37655ms step_avg:145.95ms
step:269/1480 train_time:37805ms step_avg:145.96ms
step:270/1480 train_time:37955ms step_avg:145.98ms
step:271/1480 train_time:38106ms step_avg:146.00ms
step:272/1480 train_time:38256ms step_avg:146.02ms
step:273/1480 train_time:38407ms step_avg:146.04ms
step:274/1480 train_time:38559ms step_avg:146.06ms
step:275/1480 train_time:38710ms step_avg:146.07ms
step:276/1480 train_time:38860ms step_avg:146.09ms
step:277/1480 train_time:39011ms step_avg:146.11ms
step:278/1480 train_time:39161ms step_avg:146.12ms
step:279/1480 train_time:39312ms step_avg:146.14ms
step:280/1480 train_time:39464ms step_avg:146.16ms
step:281/1480 train_time:39615ms step_avg:146.18ms
step:282/1480 train_time:39765ms step_avg:146.20ms
step:283/1480 train_time:39916ms step_avg:146.21ms
step:284/1480 train_time:40066ms step_avg:146.22ms
step:285/1480 train_time:40216ms step_avg:146.24ms
step:286/1480 train_time:40366ms step_avg:146.25ms
step:287/1480 train_time:40517ms step_avg:146.27ms
step:288/1480 train_time:40666ms step_avg:146.28ms
step:289/1480 train_time:40818ms step_avg:146.30ms
step:290/1480 train_time:40969ms step_avg:146.32ms
step:291/1480 train_time:41119ms step_avg:146.33ms
step:292/1480 train_time:41269ms step_avg:146.34ms
step:293/1480 train_time:41419ms step_avg:146.36ms
step:294/1480 train_time:41570ms step_avg:146.37ms
step:295/1480 train_time:41720ms step_avg:146.39ms
step:296/1480 train_time:41871ms step_avg:146.40ms
step:297/1480 train_time:42021ms step_avg:146.41ms
step:298/1480 train_time:42171ms step_avg:146.43ms
step:299/1480 train_time:42321ms step_avg:146.44ms
step:300/1480 train_time:42473ms step_avg:146.46ms
step:301/1480 train_time:42623ms step_avg:146.47ms
step:302/1480 train_time:42773ms step_avg:146.48ms
step:303/1480 train_time:42923ms step_avg:146.49ms
step:304/1480 train_time:43073ms step_avg:146.51ms
step:305/1480 train_time:43224ms step_avg:146.52ms
step:306/1480 train_time:43374ms step_avg:146.53ms
step:307/1480 train_time:43524ms step_avg:146.55ms
step:308/1480 train_time:43675ms step_avg:146.56ms
step:309/1480 train_time:43826ms step_avg:146.58ms
step:310/1480 train_time:43977ms step_avg:146.59ms
step:311/1480 train_time:44127ms step_avg:146.60ms
step:312/1480 train_time:44276ms step_avg:146.61ms
step:313/1480 train_time:44427ms step_avg:146.62ms
step:314/1480 train_time:44578ms step_avg:146.64ms
step:315/1480 train_time:44730ms step_avg:146.66ms
step:316/1480 train_time:44880ms step_avg:146.67ms
step:317/1480 train_time:45031ms step_avg:146.68ms
step:318/1480 train_time:45181ms step_avg:146.69ms
step:319/1480 train_time:45332ms step_avg:146.71ms
step:320/1480 train_time:45482ms step_avg:146.72ms
step:321/1480 train_time:45633ms step_avg:146.73ms
step:322/1480 train_time:45782ms step_avg:146.74ms
step:323/1480 train_time:45934ms step_avg:146.75ms
step:324/1480 train_time:46084ms step_avg:146.76ms
step:325/1480 train_time:46234ms step_avg:146.78ms
step:326/1480 train_time:46384ms step_avg:146.78ms
step:327/1480 train_time:46535ms step_avg:146.80ms
step:328/1480 train_time:46685ms step_avg:146.81ms
step:329/1480 train_time:46835ms step_avg:146.82ms
step:330/1480 train_time:46988ms step_avg:146.84ms
step:331/1480 train_time:47141ms step_avg:146.86ms
step:332/1480 train_time:47295ms step_avg:146.88ms
step:333/1480 train_time:47448ms step_avg:146.90ms
step:334/1480 train_time:47601ms step_avg:146.92ms
step:335/1480 train_time:47755ms step_avg:146.94ms
step:336/1480 train_time:47909ms step_avg:146.96ms
step:337/1480 train_time:48063ms step_avg:146.98ms
step:338/1480 train_time:48217ms step_avg:147.00ms
step:339/1480 train_time:48370ms step_avg:147.02ms
step:340/1480 train_time:48523ms step_avg:147.04ms
step:341/1480 train_time:48678ms step_avg:147.06ms
step:342/1480 train_time:48831ms step_avg:147.08ms
step:343/1480 train_time:48986ms step_avg:147.10ms
step:344/1480 train_time:49141ms step_avg:147.13ms
step:345/1480 train_time:49295ms step_avg:147.15ms
step:346/1480 train_time:49451ms step_avg:147.18ms
step:347/1480 train_time:49606ms step_avg:147.20ms
step:348/1480 train_time:49760ms step_avg:147.22ms
step:349/1480 train_time:49914ms step_avg:147.24ms
step:350/1480 train_time:50070ms step_avg:147.26ms
step:351/1480 train_time:50225ms step_avg:147.29ms
step:352/1480 train_time:50378ms step_avg:147.30ms
step:353/1480 train_time:50532ms step_avg:147.32ms
step:354/1480 train_time:50686ms step_avg:147.34ms
step:355/1480 train_time:50839ms step_avg:147.36ms
step:356/1480 train_time:50993ms step_avg:147.38ms
step:357/1480 train_time:51149ms step_avg:147.40ms
step:358/1480 train_time:51302ms step_avg:147.42ms
step:359/1480 train_time:51456ms step_avg:147.44ms
step:360/1480 train_time:51611ms step_avg:147.46ms
step:361/1480 train_time:51766ms step_avg:147.48ms
step:362/1480 train_time:51919ms step_avg:147.50ms
step:363/1480 train_time:52072ms step_avg:147.51ms
step:364/1480 train_time:52226ms step_avg:147.53ms
step:365/1480 train_time:52379ms step_avg:147.55ms
step:366/1480 train_time:52533ms step_avg:147.57ms
step:367/1480 train_time:52688ms step_avg:147.59ms
step:368/1480 train_time:52841ms step_avg:147.60ms
step:369/1480 train_time:52994ms step_avg:147.62ms
step:370/1480 train_time:53148ms step_avg:147.63ms
step:371/1480 train_time:53301ms step_avg:147.65ms
step:372/1480 train_time:53456ms step_avg:147.67ms
step:373/1480 train_time:53609ms step_avg:147.68ms
step:374/1480 train_time:53762ms step_avg:147.70ms
step:375/1480 train_time:53917ms step_avg:147.72ms
step:375/1480 val_loss:3.8107 train_time:53977ms step_avg:147.88ms
step:376/1480 train_time:54073ms step_avg:147.74ms
step:377/1480 train_time:54228ms step_avg:147.76ms
step:378/1480 train_time:54380ms step_avg:147.77ms
step:379/1480 train_time:54533ms step_avg:147.79ms
step:380/1480 train_time:54685ms step_avg:147.80ms
step:381/1480 train_time:54837ms step_avg:147.81ms
step:382/1480 train_time:54991ms step_avg:147.82ms
step:383/1480 train_time:55149ms step_avg:147.85ms
step:384/1480 train_time:55304ms step_avg:147.87ms
step:385/1480 train_time:55456ms step_avg:147.88ms
step:386/1480 train_time:55609ms step_avg:147.90ms
step:387/1480 train_time:55763ms step_avg:147.91ms
step:388/1480 train_time:55916ms step_avg:147.93ms
step:389/1480 train_time:56070ms step_avg:147.94ms
step:390/1480 train_time:56225ms step_avg:147.96ms
step:391/1480 train_time:56379ms step_avg:147.98ms
step:392/1480 train_time:56533ms step_avg:147.99ms
step:393/1480 train_time:56687ms step_avg:148.01ms
step:394/1480 train_time:56840ms step_avg:148.02ms
step:395/1480 train_time:56994ms step_avg:148.04ms
step:396/1480 train_time:57151ms step_avg:148.06ms
step:397/1480 train_time:57303ms step_avg:148.07ms
step:398/1480 train_time:57457ms step_avg:148.09ms
step:399/1480 train_time:57612ms step_avg:148.10ms
step:400/1480 train_time:57766ms step_avg:148.12ms
step:401/1480 train_time:57919ms step_avg:148.13ms
step:402/1480 train_time:58073ms step_avg:148.14ms
step:403/1480 train_time:58228ms step_avg:148.16ms
step:404/1480 train_time:58383ms step_avg:148.18ms
step:405/1480 train_time:58537ms step_avg:148.19ms
step:406/1480 train_time:58690ms step_avg:148.21ms
step:407/1480 train_time:58846ms step_avg:148.23ms
step:408/1480 train_time:58999ms step_avg:148.24ms
step:409/1480 train_time:59153ms step_avg:148.25ms
step:410/1480 train_time:59307ms step_avg:148.27ms
step:411/1480 train_time:59462ms step_avg:148.28ms
step:412/1480 train_time:59616ms step_avg:148.30ms
step:413/1480 train_time:59769ms step_avg:148.31ms
step:414/1480 train_time:59925ms step_avg:148.33ms
step:415/1480 train_time:60079ms step_avg:148.34ms
step:416/1480 train_time:60232ms step_avg:148.35ms
step:417/1480 train_time:60386ms step_avg:148.37ms
step:418/1480 train_time:60540ms step_avg:148.38ms
step:419/1480 train_time:60693ms step_avg:148.39ms
step:420/1480 train_time:60847ms step_avg:148.41ms
step:421/1480 train_time:61000ms step_avg:148.42ms
step:422/1480 train_time:61155ms step_avg:148.43ms
step:423/1480 train_time:61309ms step_avg:148.45ms
step:424/1480 train_time:61464ms step_avg:148.46ms
step:425/1480 train_time:61620ms step_avg:148.48ms
step:426/1480 train_time:61774ms step_avg:148.49ms
step:427/1480 train_time:61928ms step_avg:148.51ms
step:428/1480 train_time:62081ms step_avg:148.52ms
step:429/1480 train_time:62234ms step_avg:148.53ms
step:430/1480 train_time:62387ms step_avg:148.54ms
step:431/1480 train_time:62542ms step_avg:148.56ms
step:432/1480 train_time:62695ms step_avg:148.57ms
step:433/1480 train_time:62849ms step_avg:148.58ms
step:434/1480 train_time:63003ms step_avg:148.59ms
step:435/1480 train_time:63157ms step_avg:148.60ms
step:436/1480 train_time:63311ms step_avg:148.62ms
step:437/1480 train_time:63465ms step_avg:148.63ms
step:438/1480 train_time:63618ms step_avg:148.64ms
step:439/1480 train_time:63772ms step_avg:148.65ms
step:440/1480 train_time:63927ms step_avg:148.67ms
step:441/1480 train_time:64084ms step_avg:148.69ms
step:442/1480 train_time:64240ms step_avg:148.70ms
step:443/1480 train_time:64395ms step_avg:148.72ms
step:444/1480 train_time:64551ms step_avg:148.74ms
step:445/1480 train_time:64708ms step_avg:148.75ms
step:446/1480 train_time:64863ms step_avg:148.77ms
step:447/1480 train_time:65019ms step_avg:148.79ms
step:448/1480 train_time:65174ms step_avg:148.80ms
step:449/1480 train_time:65334ms step_avg:148.82ms
step:450/1480 train_time:65490ms step_avg:148.84ms
step:451/1480 train_time:65649ms step_avg:148.86ms
step:452/1480 train_time:65806ms step_avg:148.88ms
step:453/1480 train_time:65963ms step_avg:148.90ms
step:454/1480 train_time:66119ms step_avg:148.92ms
step:455/1480 train_time:66275ms step_avg:148.93ms
step:456/1480 train_time:66430ms step_avg:148.95ms
step:457/1480 train_time:66587ms step_avg:148.96ms
step:458/1480 train_time:66742ms step_avg:148.98ms
step:459/1480 train_time:66899ms step_avg:148.99ms
step:460/1480 train_time:67054ms step_avg:149.01ms
step:461/1480 train_time:67213ms step_avg:149.03ms
step:462/1480 train_time:67370ms step_avg:149.05ms
step:463/1480 train_time:67528ms step_avg:149.07ms
step:464/1480 train_time:67685ms step_avg:149.08ms
step:465/1480 train_time:67840ms step_avg:149.10ms
step:466/1480 train_time:67995ms step_avg:149.11ms
step:467/1480 train_time:68153ms step_avg:149.13ms
step:468/1480 train_time:68310ms step_avg:149.15ms
step:469/1480 train_time:68468ms step_avg:149.17ms
step:470/1480 train_time:68626ms step_avg:149.19ms
step:471/1480 train_time:68782ms step_avg:149.20ms
step:472/1480 train_time:68939ms step_avg:149.22ms
step:473/1480 train_time:69094ms step_avg:149.23ms
step:474/1480 train_time:69251ms step_avg:149.25ms
step:475/1480 train_time:69409ms step_avg:149.27ms
step:476/1480 train_time:69567ms step_avg:149.29ms
step:477/1480 train_time:69725ms step_avg:149.30ms
step:478/1480 train_time:69881ms step_avg:149.32ms
step:479/1480 train_time:70037ms step_avg:149.33ms
step:480/1480 train_time:70193ms step_avg:149.35ms
step:481/1480 train_time:70350ms step_avg:149.36ms
step:482/1480 train_time:70508ms step_avg:149.38ms
step:483/1480 train_time:70666ms step_avg:149.40ms
step:484/1480 train_time:70823ms step_avg:149.42ms
step:485/1480 train_time:70980ms step_avg:149.43ms
step:486/1480 train_time:71137ms step_avg:149.45ms
step:487/1480 train_time:71293ms step_avg:149.46ms
step:488/1480 train_time:71450ms step_avg:149.48ms
step:489/1480 train_time:71607ms step_avg:149.49ms
step:490/1480 train_time:71764ms step_avg:149.51ms
step:491/1480 train_time:71921ms step_avg:149.52ms
step:492/1480 train_time:72077ms step_avg:149.54ms
step:493/1480 train_time:72233ms step_avg:149.55ms
step:494/1480 train_time:72389ms step_avg:149.56ms
step:495/1480 train_time:72548ms step_avg:149.58ms
step:496/1480 train_time:72707ms step_avg:149.60ms
step:497/1480 train_time:72863ms step_avg:149.62ms
step:498/1480 train_time:73022ms step_avg:149.64ms
step:499/1480 train_time:73179ms step_avg:149.65ms
step:500/1480 train_time:73335ms step_avg:149.66ms
step:500/1480 val_loss:3.6857 train_time:73396ms step_avg:149.79ms
step:501/1480 train_time:73495ms step_avg:149.68ms
step:502/1480 train_time:73653ms step_avg:149.70ms
step:503/1480 train_time:73810ms step_avg:149.72ms
step:504/1480 train_time:73966ms step_avg:149.73ms
step:505/1480 train_time:74121ms step_avg:149.74ms
step:506/1480 train_time:74277ms step_avg:149.75ms
step:507/1480 train_time:74434ms step_avg:149.77ms
step:508/1480 train_time:74592ms step_avg:149.78ms
step:509/1480 train_time:74749ms step_avg:149.80ms
step:510/1480 train_time:74905ms step_avg:149.81ms
step:511/1480 train_time:75060ms step_avg:149.82ms
step:512/1480 train_time:75218ms step_avg:149.84ms
step:513/1480 train_time:75374ms step_avg:149.85ms
step:514/1480 train_time:75534ms step_avg:149.87ms
step:515/1480 train_time:75691ms step_avg:149.88ms
step:516/1480 train_time:75849ms step_avg:149.90ms
step:517/1480 train_time:76007ms step_avg:149.92ms
step:518/1480 train_time:76163ms step_avg:149.93ms
step:519/1480 train_time:76320ms step_avg:149.94ms
step:520/1480 train_time:76476ms step_avg:149.95ms
step:521/1480 train_time:76634ms step_avg:149.97ms
step:522/1480 train_time:76793ms step_avg:149.99ms
step:523/1480 train_time:76950ms step_avg:150.00ms
step:524/1480 train_time:77107ms step_avg:150.01ms
step:525/1480 train_time:77262ms step_avg:150.02ms
step:526/1480 train_time:77419ms step_avg:150.04ms
step:527/1480 train_time:77574ms step_avg:150.05ms
step:528/1480 train_time:77734ms step_avg:150.07ms
step:529/1480 train_time:77893ms step_avg:150.08ms
step:530/1480 train_time:78050ms step_avg:150.10ms
step:531/1480 train_time:78209ms step_avg:150.11ms
step:532/1480 train_time:78365ms step_avg:150.12ms
step:533/1480 train_time:78521ms step_avg:150.14ms
step:534/1480 train_time:78677ms step_avg:150.15ms
step:535/1480 train_time:78834ms step_avg:150.16ms
step:536/1480 train_time:78993ms step_avg:150.18ms
step:537/1480 train_time:79150ms step_avg:150.19ms
step:538/1480 train_time:79308ms step_avg:150.20ms
step:539/1480 train_time:79467ms step_avg:150.22ms
step:540/1480 train_time:79625ms step_avg:150.24ms
step:541/1480 train_time:79780ms step_avg:150.24ms
step:542/1480 train_time:79937ms step_avg:150.26ms
step:543/1480 train_time:80092ms step_avg:150.27ms
step:544/1480 train_time:80248ms step_avg:150.28ms
step:545/1480 train_time:80405ms step_avg:150.29ms
step:546/1480 train_time:80561ms step_avg:150.30ms
step:547/1480 train_time:80719ms step_avg:150.31ms
step:548/1480 train_time:80876ms step_avg:150.33ms
step:549/1480 train_time:81033ms step_avg:150.34ms
step:550/1480 train_time:81190ms step_avg:150.35ms
step:551/1480 train_time:81348ms step_avg:150.37ms
step:552/1480 train_time:81507ms step_avg:150.38ms
step:553/1480 train_time:81666ms step_avg:150.40ms
step:554/1480 train_time:81824ms step_avg:150.41ms
step:555/1480 train_time:81983ms step_avg:150.43ms
step:556/1480 train_time:82140ms step_avg:150.44ms
step:557/1480 train_time:82301ms step_avg:150.46ms
step:558/1480 train_time:82459ms step_avg:150.47ms
step:559/1480 train_time:82617ms step_avg:150.49ms
step:560/1480 train_time:82777ms step_avg:150.50ms
step:561/1480 train_time:82936ms step_avg:150.52ms
step:562/1480 train_time:83096ms step_avg:150.54ms
step:563/1480 train_time:83254ms step_avg:150.55ms
step:564/1480 train_time:83416ms step_avg:150.57ms
step:565/1480 train_time:83576ms step_avg:150.59ms
step:566/1480 train_time:83737ms step_avg:150.61ms
step:567/1480 train_time:83896ms step_avg:150.62ms
step:568/1480 train_time:84055ms step_avg:150.64ms
step:569/1480 train_time:84215ms step_avg:150.65ms
step:570/1480 train_time:84374ms step_avg:150.67ms
step:571/1480 train_time:84535ms step_avg:150.69ms
step:572/1480 train_time:84695ms step_avg:150.70ms
step:573/1480 train_time:84855ms step_avg:150.72ms
step:574/1480 train_time:85017ms step_avg:150.74ms
step:575/1480 train_time:85176ms step_avg:150.75ms
step:576/1480 train_time:85336ms step_avg:150.77ms
step:577/1480 train_time:85495ms step_avg:150.79ms
step:578/1480 train_time:85654ms step_avg:150.80ms
step:579/1480 train_time:85815ms step_avg:150.82ms
step:580/1480 train_time:85974ms step_avg:150.83ms
step:581/1480 train_time:86136ms step_avg:150.85ms
step:582/1480 train_time:86296ms step_avg:150.87ms
step:583/1480 train_time:86455ms step_avg:150.88ms
step:584/1480 train_time:86615ms step_avg:150.90ms
step:585/1480 train_time:86775ms step_avg:150.91ms
step:586/1480 train_time:86936ms step_avg:150.93ms
step:587/1480 train_time:87095ms step_avg:150.94ms
step:588/1480 train_time:87254ms step_avg:150.96ms
step:589/1480 train_time:87414ms step_avg:150.97ms
step:590/1480 train_time:87575ms step_avg:150.99ms
step:591/1480 train_time:87734ms step_avg:151.01ms
step:592/1480 train_time:87894ms step_avg:151.02ms
step:593/1480 train_time:88056ms step_avg:151.04ms
step:594/1480 train_time:88217ms step_avg:151.06ms
step:595/1480 train_time:88377ms step_avg:151.07ms
step:596/1480 train_time:88538ms step_avg:151.09ms
step:597/1480 train_time:88697ms step_avg:151.10ms
step:598/1480 train_time:88855ms step_avg:151.11ms
step:599/1480 train_time:89015ms step_avg:151.13ms
step:600/1480 train_time:89175ms step_avg:151.14ms
step:601/1480 train_time:89335ms step_avg:151.16ms
step:602/1480 train_time:89495ms step_avg:151.17ms
step:603/1480 train_time:89655ms step_avg:151.19ms
step:604/1480 train_time:89815ms step_avg:151.20ms
step:605/1480 train_time:89974ms step_avg:151.22ms
step:606/1480 train_time:90136ms step_avg:151.24ms
step:607/1480 train_time:90297ms step_avg:151.25ms
step:608/1480 train_time:90456ms step_avg:151.26ms
step:609/1480 train_time:90616ms step_avg:151.28ms
step:610/1480 train_time:90775ms step_avg:151.29ms
step:611/1480 train_time:90937ms step_avg:151.31ms
step:612/1480 train_time:91097ms step_avg:151.32ms
step:613/1480 train_time:91257ms step_avg:151.34ms
step:614/1480 train_time:91416ms step_avg:151.35ms
step:615/1480 train_time:91575ms step_avg:151.36ms
step:616/1480 train_time:91735ms step_avg:151.38ms
step:617/1480 train_time:91896ms step_avg:151.39ms
step:618/1480 train_time:92056ms step_avg:151.41ms
step:619/1480 train_time:92216ms step_avg:151.42ms
step:620/1480 train_time:92375ms step_avg:151.43ms
step:621/1480 train_time:92537ms step_avg:151.45ms
step:622/1480 train_time:92697ms step_avg:151.47ms
step:623/1480 train_time:92857ms step_avg:151.48ms
step:624/1480 train_time:93015ms step_avg:151.49ms
step:625/1480 train_time:93175ms step_avg:151.50ms
step:625/1480 val_loss:3.6062 train_time:93240ms step_avg:151.61ms
step:626/1480 train_time:93338ms step_avg:151.52ms
step:627/1480 train_time:93497ms step_avg:151.53ms
step:628/1480 train_time:93654ms step_avg:151.54ms
step:629/1480 train_time:93812ms step_avg:151.55ms
step:630/1480 train_time:93971ms step_avg:151.57ms
step:631/1480 train_time:94129ms step_avg:151.58ms
step:632/1480 train_time:94287ms step_avg:151.59ms
step:633/1480 train_time:94447ms step_avg:151.60ms
step:634/1480 train_time:94609ms step_avg:151.62ms
step:635/1480 train_time:94769ms step_avg:151.63ms
step:636/1480 train_time:94930ms step_avg:151.64ms
step:637/1480 train_time:95089ms step_avg:151.66ms
step:638/1480 train_time:95248ms step_avg:151.67ms
step:639/1480 train_time:95407ms step_avg:151.68ms
step:640/1480 train_time:95568ms step_avg:151.70ms
step:641/1480 train_time:95728ms step_avg:151.71ms
step:642/1480 train_time:95889ms step_avg:151.72ms
step:643/1480 train_time:96049ms step_avg:151.74ms
step:644/1480 train_time:96208ms step_avg:151.75ms
step:645/1480 train_time:96367ms step_avg:151.76ms
step:646/1480 train_time:96527ms step_avg:151.77ms
step:647/1480 train_time:96688ms step_avg:151.79ms
step:648/1480 train_time:96849ms step_avg:151.80ms
step:649/1480 train_time:97009ms step_avg:151.81ms
step:650/1480 train_time:97170ms step_avg:151.83ms
step:651/1480 train_time:97330ms step_avg:151.84ms
step:652/1480 train_time:97489ms step_avg:151.85ms
step:653/1480 train_time:97648ms step_avg:151.86ms
step:654/1480 train_time:97808ms step_avg:151.88ms
step:655/1480 train_time:97968ms step_avg:151.89ms
step:656/1480 train_time:98128ms step_avg:151.90ms
step:657/1480 train_time:98290ms step_avg:151.92ms
step:658/1480 train_time:98449ms step_avg:151.93ms
step:659/1480 train_time:98611ms step_avg:151.94ms
step:660/1480 train_time:98772ms step_avg:151.96ms
step:661/1480 train_time:98934ms step_avg:151.97ms
step:662/1480 train_time:99094ms step_avg:151.98ms
step:663/1480 train_time:99252ms step_avg:151.99ms
step:664/1480 train_time:99414ms step_avg:152.01ms
step:665/1480 train_time:99577ms step_avg:152.03ms
step:666/1480 train_time:99736ms step_avg:152.04ms
step:667/1480 train_time:99898ms step_avg:152.05ms
step:668/1480 train_time:100059ms step_avg:152.07ms
step:669/1480 train_time:100221ms step_avg:152.08ms
step:670/1480 train_time:100381ms step_avg:152.09ms
step:671/1480 train_time:100543ms step_avg:152.11ms
step:672/1480 train_time:100706ms step_avg:152.12ms
step:673/1480 train_time:100869ms step_avg:152.14ms
step:674/1480 train_time:101032ms step_avg:152.16ms
step:675/1480 train_time:101195ms step_avg:152.17ms
step:676/1480 train_time:101355ms step_avg:152.19ms
step:677/1480 train_time:101515ms step_avg:152.20ms
step:678/1480 train_time:101675ms step_avg:152.21ms
step:679/1480 train_time:101836ms step_avg:152.22ms
step:680/1480 train_time:102000ms step_avg:152.24ms
step:681/1480 train_time:102163ms step_avg:152.26ms
step:682/1480 train_time:102325ms step_avg:152.27ms
step:683/1480 train_time:102488ms step_avg:152.29ms
step:684/1480 train_time:102649ms step_avg:152.30ms
step:685/1480 train_time:102814ms step_avg:152.32ms
step:686/1480 train_time:102976ms step_avg:152.33ms
step:687/1480 train_time:103136ms step_avg:152.34ms
step:688/1480 train_time:103298ms step_avg:152.36ms
step:689/1480 train_time:103460ms step_avg:152.37ms
step:690/1480 train_time:103622ms step_avg:152.39ms
step:691/1480 train_time:103783ms step_avg:152.40ms
step:692/1480 train_time:103944ms step_avg:152.41ms
step:693/1480 train_time:104107ms step_avg:152.43ms
step:694/1480 train_time:104270ms step_avg:152.44ms
step:695/1480 train_time:104431ms step_avg:152.45ms
step:696/1480 train_time:104592ms step_avg:152.47ms
step:697/1480 train_time:104753ms step_avg:152.48ms
step:698/1480 train_time:104914ms step_avg:152.49ms
step:699/1480 train_time:105077ms step_avg:152.51ms
step:700/1480 train_time:105238ms step_avg:152.52ms
step:701/1480 train_time:105398ms step_avg:152.53ms
step:702/1480 train_time:105559ms step_avg:152.54ms
step:703/1480 train_time:105718ms step_avg:152.55ms
step:704/1480 train_time:105878ms step_avg:152.56ms
step:705/1480 train_time:106042ms step_avg:152.58ms
step:706/1480 train_time:106207ms step_avg:152.60ms
step:707/1480 train_time:106369ms step_avg:152.61ms
step:708/1480 train_time:106529ms step_avg:152.62ms
step:709/1480 train_time:106691ms step_avg:152.63ms
step:710/1480 train_time:106851ms step_avg:152.64ms
step:711/1480 train_time:107013ms step_avg:152.66ms
step:712/1480 train_time:107176ms step_avg:152.67ms
step:713/1480 train_time:107341ms step_avg:152.69ms
step:714/1480 train_time:107502ms step_avg:152.70ms
step:715/1480 train_time:107663ms step_avg:152.71ms
step:716/1480 train_time:107823ms step_avg:152.72ms
step:717/1480 train_time:107988ms step_avg:152.74ms
step:718/1480 train_time:108150ms step_avg:152.75ms
step:719/1480 train_time:108310ms step_avg:152.76ms
step:720/1480 train_time:108473ms step_avg:152.78ms
step:721/1480 train_time:108634ms step_avg:152.79ms
step:722/1480 train_time:108796ms step_avg:152.80ms
step:723/1480 train_time:108955ms step_avg:152.81ms
step:724/1480 train_time:109116ms step_avg:152.82ms
step:725/1480 train_time:109279ms step_avg:152.84ms
step:726/1480 train_time:109442ms step_avg:152.85ms
step:727/1480 train_time:109607ms step_avg:152.87ms
step:728/1480 train_time:109769ms step_avg:152.88ms
step:729/1480 train_time:109931ms step_avg:152.89ms
step:730/1480 train_time:110094ms step_avg:152.91ms
step:731/1480 train_time:110254ms step_avg:152.92ms
step:732/1480 train_time:110414ms step_avg:152.93ms
step:733/1480 train_time:110574ms step_avg:152.94ms
step:734/1480 train_time:110736ms step_avg:152.95ms
step:735/1480 train_time:110896ms step_avg:152.96ms
step:736/1480 train_time:111057ms step_avg:152.97ms
step:737/1480 train_time:111216ms step_avg:152.98ms
step:738/1480 train_time:111377ms step_avg:152.99ms
step:739/1480 train_time:111535ms step_avg:153.00ms
step:740/1480 train_time:111700ms step_avg:153.01ms
step:741/1480 train_time:111864ms step_avg:153.03ms
step:742/1480 train_time:112027ms step_avg:153.04ms
step:743/1480 train_time:112190ms step_avg:153.06ms
step:744/1480 train_time:112354ms step_avg:153.07ms
step:745/1480 train_time:112517ms step_avg:153.08ms
step:746/1480 train_time:112676ms step_avg:153.09ms
step:747/1480 train_time:112837ms step_avg:153.10ms
step:748/1480 train_time:113005ms step_avg:153.12ms
step:749/1480 train_time:113168ms step_avg:153.14ms
step:750/1480 train_time:113329ms step_avg:153.15ms
step:750/1480 val_loss:3.5497 train_time:113394ms step_avg:153.24ms
step:751/1480 train_time:113494ms step_avg:153.16ms
step:752/1480 train_time:113656ms step_avg:153.17ms
step:753/1480 train_time:113816ms step_avg:153.18ms
step:754/1480 train_time:113976ms step_avg:153.19ms
step:755/1480 train_time:114137ms step_avg:153.20ms
step:756/1480 train_time:114298ms step_avg:153.21ms
step:757/1480 train_time:114464ms step_avg:153.23ms
step:758/1480 train_time:114625ms step_avg:153.24ms
step:759/1480 train_time:114788ms step_avg:153.25ms
step:760/1480 train_time:114950ms step_avg:153.27ms
step:761/1480 train_time:115113ms step_avg:153.28ms
step:762/1480 train_time:115274ms step_avg:153.29ms
step:763/1480 train_time:115436ms step_avg:153.30ms
step:764/1480 train_time:115597ms step_avg:153.31ms
step:765/1480 train_time:115758ms step_avg:153.32ms
step:766/1480 train_time:115920ms step_avg:153.33ms
step:767/1480 train_time:116081ms step_avg:153.34ms
step:768/1480 train_time:116243ms step_avg:153.36ms
step:769/1480 train_time:116407ms step_avg:153.37ms
step:770/1480 train_time:116572ms step_avg:153.38ms
step:771/1480 train_time:116736ms step_avg:153.40ms
step:772/1480 train_time:116898ms step_avg:153.41ms
step:773/1480 train_time:117058ms step_avg:153.42ms
step:774/1480 train_time:117218ms step_avg:153.43ms
step:775/1480 train_time:117379ms step_avg:153.44ms
step:776/1480 train_time:117545ms step_avg:153.45ms
step:777/1480 train_time:117713ms step_avg:153.47ms
step:778/1480 train_time:117876ms step_avg:153.48ms
step:779/1480 train_time:118038ms step_avg:153.50ms
step:780/1480 train_time:118201ms step_avg:153.51ms
step:781/1480 train_time:118363ms step_avg:153.52ms
step:782/1480 train_time:118527ms step_avg:153.53ms
step:783/1480 train_time:118689ms step_avg:153.54ms
step:784/1480 train_time:118854ms step_avg:153.56ms
step:785/1480 train_time:119016ms step_avg:153.57ms
step:786/1480 train_time:119179ms step_avg:153.58ms
step:787/1480 train_time:119341ms step_avg:153.59ms
step:788/1480 train_time:119507ms step_avg:153.61ms
step:789/1480 train_time:119670ms step_avg:153.62ms
step:790/1480 train_time:119835ms step_avg:153.63ms
step:791/1480 train_time:120003ms step_avg:153.65ms
step:792/1480 train_time:120170ms step_avg:153.67ms
step:793/1480 train_time:120333ms step_avg:153.68ms
step:794/1480 train_time:120497ms step_avg:153.69ms
step:795/1480 train_time:120663ms step_avg:153.71ms
step:796/1480 train_time:120831ms step_avg:153.73ms
step:797/1480 train_time:120994ms step_avg:153.74ms
step:798/1480 train_time:121158ms step_avg:153.75ms
step:799/1480 train_time:121326ms step_avg:153.77ms
step:800/1480 train_time:121490ms step_avg:153.78ms
step:801/1480 train_time:121653ms step_avg:153.80ms
step:802/1480 train_time:121819ms step_avg:153.81ms
step:803/1480 train_time:121980ms step_avg:153.82ms
step:804/1480 train_time:122141ms step_avg:153.83ms
step:805/1480 train_time:122306ms step_avg:153.84ms
step:806/1480 train_time:122469ms step_avg:153.86ms
step:807/1480 train_time:122630ms step_avg:153.86ms
step:808/1480 train_time:122794ms step_avg:153.88ms
step:809/1480 train_time:122956ms step_avg:153.89ms
step:810/1480 train_time:123117ms step_avg:153.90ms
step:811/1480 train_time:123279ms step_avg:153.91ms
step:812/1480 train_time:123442ms step_avg:153.92ms
step:813/1480 train_time:123603ms step_avg:153.93ms
step:814/1480 train_time:123768ms step_avg:153.94ms
step:815/1480 train_time:123931ms step_avg:153.95ms
step:816/1480 train_time:124095ms step_avg:153.96ms
step:817/1480 train_time:124257ms step_avg:153.97ms
step:818/1480 train_time:124417ms step_avg:153.98ms
step:819/1480 train_time:124580ms step_avg:153.99ms
step:820/1480 train_time:124744ms step_avg:154.01ms
step:821/1480 train_time:124905ms step_avg:154.01ms
step:822/1480 train_time:125070ms step_avg:154.03ms
step:823/1480 train_time:125233ms step_avg:154.04ms
step:824/1480 train_time:125395ms step_avg:154.05ms
step:825/1480 train_time:125559ms step_avg:154.06ms
step:826/1480 train_time:125726ms step_avg:154.08ms
step:827/1480 train_time:125892ms step_avg:154.09ms
step:828/1480 train_time:126055ms step_avg:154.10ms
step:829/1480 train_time:126217ms step_avg:154.11ms
step:830/1480 train_time:126380ms step_avg:154.12ms
step:831/1480 train_time:126544ms step_avg:154.13ms
step:832/1480 train_time:126708ms step_avg:154.15ms
step:833/1480 train_time:126874ms step_avg:154.16ms
step:834/1480 train_time:127038ms step_avg:154.17ms
step:835/1480 train_time:127200ms step_avg:154.18ms
step:836/1480 train_time:127368ms step_avg:154.20ms
step:837/1480 train_time:127531ms step_avg:154.21ms
step:838/1480 train_time:127695ms step_avg:154.22ms
step:839/1480 train_time:127856ms step_avg:154.23ms
step:840/1480 train_time:128017ms step_avg:154.24ms
step:841/1480 train_time:128177ms step_avg:154.24ms
step:842/1480 train_time:128339ms step_avg:154.25ms
step:843/1480 train_time:128500ms step_avg:154.26ms
step:844/1480 train_time:128665ms step_avg:154.27ms
step:845/1480 train_time:128829ms step_avg:154.29ms
step:846/1480 train_time:128992ms step_avg:154.30ms
step:847/1480 train_time:129155ms step_avg:154.31ms
step:848/1480 train_time:129316ms step_avg:154.32ms
step:849/1480 train_time:129478ms step_avg:154.32ms
step:850/1480 train_time:129641ms step_avg:154.33ms
step:851/1480 train_time:129805ms step_avg:154.35ms
step:852/1480 train_time:129969ms step_avg:154.36ms
step:853/1480 train_time:130132ms step_avg:154.37ms
step:854/1480 train_time:130295ms step_avg:154.38ms
step:855/1480 train_time:130458ms step_avg:154.39ms
step:856/1480 train_time:130620ms step_avg:154.40ms
step:857/1480 train_time:130785ms step_avg:154.41ms
step:858/1480 train_time:130951ms step_avg:154.42ms
step:859/1480 train_time:131115ms step_avg:154.43ms
step:860/1480 train_time:131276ms step_avg:154.44ms
step:861/1480 train_time:131444ms step_avg:154.46ms
step:862/1480 train_time:131613ms step_avg:154.48ms
step:863/1480 train_time:131779ms step_avg:154.49ms
step:864/1480 train_time:131941ms step_avg:154.50ms
step:865/1480 train_time:132103ms step_avg:154.51ms
step:866/1480 train_time:132272ms step_avg:154.52ms
step:867/1480 train_time:132435ms step_avg:154.53ms
step:868/1480 train_time:132596ms step_avg:154.54ms
step:869/1480 train_time:132758ms step_avg:154.55ms
step:870/1480 train_time:132923ms step_avg:154.56ms
step:871/1480 train_time:133085ms step_avg:154.57ms
step:872/1480 train_time:133251ms step_avg:154.58ms
step:873/1480 train_time:133413ms step_avg:154.59ms
step:874/1480 train_time:133579ms step_avg:154.61ms
step:875/1480 train_time:133743ms step_avg:154.62ms
step:875/1480 val_loss:3.5062 train_time:133808ms step_avg:154.69ms
step:876/1480 train_time:133907ms step_avg:154.63ms
step:877/1480 train_time:134073ms step_avg:154.64ms
step:878/1480 train_time:134236ms step_avg:154.65ms
step:879/1480 train_time:134401ms step_avg:154.66ms
step:880/1480 train_time:134564ms step_avg:154.67ms
step:881/1480 train_time:134727ms step_avg:154.68ms
step:882/1480 train_time:134891ms step_avg:154.69ms
step:883/1480 train_time:135057ms step_avg:154.70ms
step:884/1480 train_time:135225ms step_avg:154.72ms
step:885/1480 train_time:135389ms step_avg:154.73ms
step:886/1480 train_time:135555ms step_avg:154.74ms
step:887/1480 train_time:135724ms step_avg:154.76ms
step:888/1480 train_time:135898ms step_avg:154.78ms
step:889/1480 train_time:136066ms step_avg:154.80ms
step:890/1480 train_time:136228ms step_avg:154.80ms
step:891/1480 train_time:136394ms step_avg:154.82ms
step:892/1480 train_time:136559ms step_avg:154.83ms
step:893/1480 train_time:136723ms step_avg:154.84ms
step:894/1480 train_time:136888ms step_avg:154.85ms
step:895/1480 train_time:137053ms step_avg:154.86ms
step:896/1480 train_time:137218ms step_avg:154.87ms
step:897/1480 train_time:137385ms step_avg:154.89ms
step:898/1480 train_time:137551ms step_avg:154.90ms
step:899/1480 train_time:137715ms step_avg:154.91ms
step:900/1480 train_time:137878ms step_avg:154.92ms
step:901/1480 train_time:138044ms step_avg:154.93ms
step:902/1480 train_time:138207ms step_avg:154.94ms
step:903/1480 train_time:138379ms step_avg:154.96ms
step:904/1480 train_time:138545ms step_avg:154.97ms
step:905/1480 train_time:138707ms step_avg:154.98ms
step:906/1480 train_time:138873ms step_avg:154.99ms
step:907/1480 train_time:139042ms step_avg:155.01ms
step:908/1480 train_time:139205ms step_avg:155.02ms
step:909/1480 train_time:139369ms step_avg:155.03ms
step:910/1480 train_time:139539ms step_avg:155.04ms
step:911/1480 train_time:139704ms step_avg:155.05ms
step:912/1480 train_time:139870ms step_avg:155.07ms
step:913/1480 train_time:140038ms step_avg:155.08ms
step:914/1480 train_time:140206ms step_avg:155.10ms
step:915/1480 train_time:140375ms step_avg:155.11ms
step:916/1480 train_time:140541ms step_avg:155.12ms
step:917/1480 train_time:140705ms step_avg:155.13ms
step:918/1480 train_time:140872ms step_avg:155.15ms
step:919/1480 train_time:141042ms step_avg:155.16ms
step:920/1480 train_time:141206ms step_avg:155.17ms
step:921/1480 train_time:141372ms step_avg:155.18ms
step:922/1480 train_time:141539ms step_avg:155.20ms
step:923/1480 train_time:141702ms step_avg:155.20ms
step:924/1480 train_time:141866ms step_avg:155.21ms
step:925/1480 train_time:142031ms step_avg:155.22ms
step:926/1480 train_time:142193ms step_avg:155.23ms
step:927/1480 train_time:142356ms step_avg:155.24ms
step:928/1480 train_time:142524ms step_avg:155.25ms
step:929/1480 train_time:142688ms step_avg:155.26ms
step:930/1480 train_time:142854ms step_avg:155.28ms
step:931/1480 train_time:143018ms step_avg:155.29ms
step:932/1480 train_time:143185ms step_avg:155.30ms
step:933/1480 train_time:143352ms step_avg:155.31ms
step:934/1480 train_time:143521ms step_avg:155.33ms
step:935/1480 train_time:143691ms step_avg:155.34ms
step:936/1480 train_time:143858ms step_avg:155.35ms
step:937/1480 train_time:144028ms step_avg:155.37ms
step:938/1480 train_time:144189ms step_avg:155.38ms
step:939/1480 train_time:144359ms step_avg:155.39ms
step:940/1480 train_time:144527ms step_avg:155.41ms
step:941/1480 train_time:144690ms step_avg:155.41ms
step:942/1480 train_time:144855ms step_avg:155.42ms
step:943/1480 train_time:145027ms step_avg:155.44ms
step:944/1480 train_time:145199ms step_avg:155.46ms
step:945/1480 train_time:145362ms step_avg:155.47ms
step:946/1480 train_time:145531ms step_avg:155.48ms
step:947/1480 train_time:145701ms step_avg:155.50ms
step:948/1480 train_time:145866ms step_avg:155.51ms
step:949/1480 train_time:146030ms step_avg:155.52ms
step:950/1480 train_time:146193ms step_avg:155.52ms
step:951/1480 train_time:146363ms step_avg:155.54ms
step:952/1480 train_time:146528ms step_avg:155.55ms
step:953/1480 train_time:146695ms step_avg:155.56ms
step:954/1480 train_time:146865ms step_avg:155.58ms
step:955/1480 train_time:147028ms step_avg:155.59ms
step:956/1480 train_time:147195ms step_avg:155.60ms
step:957/1480 train_time:147363ms step_avg:155.61ms
step:958/1480 train_time:147532ms step_avg:155.62ms
step:959/1480 train_time:147696ms step_avg:155.63ms
step:960/1480 train_time:147863ms step_avg:155.65ms
step:961/1480 train_time:148028ms step_avg:155.65ms
step:962/1480 train_time:148191ms step_avg:155.66ms
step:963/1480 train_time:148357ms step_avg:155.67ms
step:964/1480 train_time:148526ms step_avg:155.69ms
step:965/1480 train_time:148689ms step_avg:155.70ms
step:966/1480 train_time:148853ms step_avg:155.70ms
step:967/1480 train_time:149018ms step_avg:155.71ms
step:968/1480 train_time:149183ms step_avg:155.72ms
step:969/1480 train_time:149348ms step_avg:155.73ms
step:970/1480 train_time:149513ms step_avg:155.74ms
step:971/1480 train_time:149678ms step_avg:155.75ms
step:972/1480 train_time:149842ms step_avg:155.76ms
step:973/1480 train_time:150007ms step_avg:155.77ms
step:974/1480 train_time:150174ms step_avg:155.78ms
step:975/1480 train_time:150339ms step_avg:155.79ms
step:976/1480 train_time:150504ms step_avg:155.80ms
step:977/1480 train_time:150667ms step_avg:155.81ms
step:978/1480 train_time:150831ms step_avg:155.82ms
step:979/1480 train_time:150997ms step_avg:155.83ms
step:980/1480 train_time:151163ms step_avg:155.84ms
step:981/1480 train_time:151330ms step_avg:155.85ms
step:982/1480 train_time:151494ms step_avg:155.86ms
step:983/1480 train_time:151661ms step_avg:155.87ms
step:984/1480 train_time:151825ms step_avg:155.88ms
step:985/1480 train_time:151991ms step_avg:155.89ms
step:986/1480 train_time:152156ms step_avg:155.90ms
step:987/1480 train_time:152322ms step_avg:155.91ms
step:988/1480 train_time:152489ms step_avg:155.92ms
step:989/1480 train_time:152655ms step_avg:155.93ms
step:990/1480 train_time:152825ms step_avg:155.94ms
step:991/1480 train_time:152992ms step_avg:155.96ms
step:992/1480 train_time:153167ms step_avg:155.97ms
step:993/1480 train_time:153344ms step_avg:156.00ms
step:994/1480 train_time:153508ms step_avg:156.00ms
step:995/1480 train_time:153672ms step_avg:156.01ms
step:996/1480 train_time:153835ms step_avg:156.02ms
step:997/1480 train_time:154001ms step_avg:156.03ms
step:998/1480 train_time:154165ms step_avg:156.04ms
step:999/1480 train_time:154329ms step_avg:156.05ms
step:1000/1480 train_time:154500ms step_avg:156.06ms
step:1000/1480 val_loss:3.4443 train_time:154567ms step_avg:156.13ms
step:1001/1480 train_time:154667ms step_avg:156.07ms
step:1002/1480 train_time:154833ms step_avg:156.08ms
step:1003/1480 train_time:155005ms step_avg:156.10ms
step:1004/1480 train_time:155173ms step_avg:156.11ms
step:1005/1480 train_time:155341ms step_avg:156.12ms
step:1006/1480 train_time:155508ms step_avg:156.13ms
step:1007/1480 train_time:155673ms step_avg:156.14ms
step:1008/1480 train_time:155843ms step_avg:156.16ms
step:1009/1480 train_time:156017ms step_avg:156.17ms
step:1010/1480 train_time:156183ms step_avg:156.18ms
step:1011/1480 train_time:156349ms step_avg:156.19ms
step:1012/1480 train_time:156512ms step_avg:156.20ms
step:1013/1480 train_time:156683ms step_avg:156.21ms
step:1014/1480 train_time:156849ms step_avg:156.22ms
step:1015/1480 train_time:157021ms step_avg:156.24ms
step:1016/1480 train_time:157189ms step_avg:156.25ms
step:1017/1480 train_time:157360ms step_avg:156.27ms
step:1018/1480 train_time:157527ms step_avg:156.28ms
step:1019/1480 train_time:157697ms step_avg:156.29ms
step:1020/1480 train_time:157867ms step_avg:156.30ms
step:1021/1480 train_time:158030ms step_avg:156.31ms
step:1022/1480 train_time:158197ms step_avg:156.32ms
step:1023/1480 train_time:158363ms step_avg:156.33ms
step:1024/1480 train_time:158529ms step_avg:156.34ms
step:1025/1480 train_time:158700ms step_avg:156.35ms
step:1026/1480 train_time:158867ms step_avg:156.36ms
step:1027/1480 train_time:159032ms step_avg:156.37ms
step:1028/1480 train_time:159205ms step_avg:156.39ms
step:1029/1480 train_time:159379ms step_avg:156.41ms
step:1030/1480 train_time:159547ms step_avg:156.42ms
step:1031/1480 train_time:159711ms step_avg:156.43ms
step:1032/1480 train_time:159886ms step_avg:156.44ms
step:1033/1480 train_time:160051ms step_avg:156.45ms
step:1034/1480 train_time:160220ms step_avg:156.47ms
step:1035/1480 train_time:160387ms step_avg:156.48ms
step:1036/1480 train_time:160553ms step_avg:156.48ms
step:1037/1480 train_time:160721ms step_avg:156.50ms
step:1038/1480 train_time:160888ms step_avg:156.51ms
step:1039/1480 train_time:161060ms step_avg:156.52ms
step:1040/1480 train_time:161225ms step_avg:156.53ms
step:1041/1480 train_time:161391ms step_avg:156.54ms
step:1042/1480 train_time:161555ms step_avg:156.55ms
step:1043/1480 train_time:161721ms step_avg:156.56ms
step:1044/1480 train_time:161887ms step_avg:156.56ms
step:1045/1480 train_time:162058ms step_avg:156.58ms
step:1046/1480 train_time:162226ms step_avg:156.59ms
step:1047/1480 train_time:162391ms step_avg:156.60ms
step:1048/1480 train_time:162557ms step_avg:156.61ms
step:1049/1480 train_time:162722ms step_avg:156.61ms
step:1050/1480 train_time:162891ms step_avg:156.63ms
step:1051/1480 train_time:163061ms step_avg:156.64ms
step:1052/1480 train_time:163229ms step_avg:156.65ms
step:1053/1480 train_time:163395ms step_avg:156.66ms
step:1054/1480 train_time:163565ms step_avg:156.67ms
step:1055/1480 train_time:163730ms step_avg:156.68ms
step:1056/1480 train_time:163895ms step_avg:156.69ms
step:1057/1480 train_time:164063ms step_avg:156.70ms
step:1058/1480 train_time:164232ms step_avg:156.71ms
step:1059/1480 train_time:164404ms step_avg:156.72ms
step:1060/1480 train_time:164572ms step_avg:156.74ms
step:1061/1480 train_time:164735ms step_avg:156.74ms
step:1062/1480 train_time:164902ms step_avg:156.75ms
step:1063/1480 train_time:165067ms step_avg:156.76ms
step:1064/1480 train_time:165231ms step_avg:156.77ms
step:1065/1480 train_time:165398ms step_avg:156.78ms
step:1066/1480 train_time:165565ms step_avg:156.79ms
step:1067/1480 train_time:165735ms step_avg:156.80ms
step:1068/1480 train_time:165901ms step_avg:156.81ms
step:1069/1480 train_time:166071ms step_avg:156.82ms
step:1070/1480 train_time:166237ms step_avg:156.83ms
step:1071/1480 train_time:166409ms step_avg:156.84ms
step:1072/1480 train_time:166576ms step_avg:156.85ms
step:1073/1480 train_time:166740ms step_avg:156.86ms
step:1074/1480 train_time:166906ms step_avg:156.87ms
step:1075/1480 train_time:167075ms step_avg:156.88ms
step:1076/1480 train_time:167245ms step_avg:156.89ms
step:1077/1480 train_time:167412ms step_avg:156.90ms
step:1078/1480 train_time:167586ms step_avg:156.92ms
step:1079/1480 train_time:167760ms step_avg:156.93ms
step:1080/1480 train_time:167929ms step_avg:156.94ms
step:1081/1480 train_time:168096ms step_avg:156.95ms
step:1082/1480 train_time:168262ms step_avg:156.96ms
step:1083/1480 train_time:168429ms step_avg:156.97ms
step:1084/1480 train_time:168594ms step_avg:156.98ms
step:1085/1480 train_time:168764ms step_avg:156.99ms
step:1086/1480 train_time:168932ms step_avg:157.00ms
step:1087/1480 train_time:169097ms step_avg:157.01ms
step:1088/1480 train_time:169268ms step_avg:157.02ms
step:1089/1480 train_time:169440ms step_avg:157.03ms
step:1090/1480 train_time:169611ms step_avg:157.05ms
step:1091/1480 train_time:169780ms step_avg:157.06ms
step:1092/1480 train_time:169946ms step_avg:157.07ms
step:1093/1480 train_time:170114ms step_avg:157.08ms
step:1094/1480 train_time:170281ms step_avg:157.09ms
step:1095/1480 train_time:170446ms step_avg:157.09ms
step:1096/1480 train_time:170615ms step_avg:157.10ms
step:1097/1480 train_time:170784ms step_avg:157.11ms
step:1098/1480 train_time:170954ms step_avg:157.13ms
step:1099/1480 train_time:171125ms step_avg:157.14ms
step:1100/1480 train_time:171296ms step_avg:157.15ms
step:1101/1480 train_time:171466ms step_avg:157.16ms
step:1102/1480 train_time:171638ms step_avg:157.18ms
step:1103/1480 train_time:171815ms step_avg:157.20ms
step:1104/1480 train_time:171983ms step_avg:157.21ms
step:1105/1480 train_time:172151ms step_avg:157.22ms
step:1106/1480 train_time:172321ms step_avg:157.23ms
step:1107/1480 train_time:172490ms step_avg:157.24ms
step:1108/1480 train_time:172655ms step_avg:157.24ms
step:1109/1480 train_time:172823ms step_avg:157.25ms
step:1110/1480 train_time:172988ms step_avg:157.26ms
step:1111/1480 train_time:173155ms step_avg:157.27ms
step:1112/1480 train_time:173326ms step_avg:157.28ms
step:1113/1480 train_time:173506ms step_avg:157.30ms
step:1114/1480 train_time:173680ms step_avg:157.32ms
step:1115/1480 train_time:173851ms step_avg:157.33ms
step:1116/1480 train_time:174017ms step_avg:157.34ms
step:1117/1480 train_time:174190ms step_avg:157.35ms
step:1118/1480 train_time:174366ms step_avg:157.37ms
step:1119/1480 train_time:174532ms step_avg:157.38ms
step:1120/1480 train_time:174701ms step_avg:157.39ms
step:1121/1480 train_time:174870ms step_avg:157.40ms
step:1122/1480 train_time:175037ms step_avg:157.41ms
step:1123/1480 train_time:175204ms step_avg:157.42ms
step:1124/1480 train_time:175373ms step_avg:157.43ms
step:1125/1480 train_time:175542ms step_avg:157.44ms
step:1125/1480 val_loss:3.3886 train_time:175611ms step_avg:157.50ms
step:1126/1480 train_time:175714ms step_avg:157.45ms
step:1127/1480 train_time:175884ms step_avg:157.46ms
step:1128/1480 train_time:176054ms step_avg:157.47ms
step:1129/1480 train_time:176226ms step_avg:157.49ms
step:1130/1480 train_time:176397ms step_avg:157.50ms
step:1131/1480 train_time:176573ms step_avg:157.51ms
step:1132/1480 train_time:176739ms step_avg:157.52ms
step:1133/1480 train_time:176910ms step_avg:157.53ms
step:1134/1480 train_time:177080ms step_avg:157.54ms
step:1135/1480 train_time:177249ms step_avg:157.55ms
step:1136/1480 train_time:177419ms step_avg:157.57ms
step:1137/1480 train_time:177588ms step_avg:157.58ms
step:1138/1480 train_time:177759ms step_avg:157.59ms
step:1139/1480 train_time:177925ms step_avg:157.60ms
step:1140/1480 train_time:178094ms step_avg:157.61ms
step:1141/1480 train_time:178265ms step_avg:157.62ms
step:1142/1480 train_time:178432ms step_avg:157.63ms
step:1143/1480 train_time:178602ms step_avg:157.64ms
step:1144/1480 train_time:178771ms step_avg:157.65ms
step:1145/1480 train_time:178938ms step_avg:157.65ms
step:1146/1480 train_time:179107ms step_avg:157.66ms
step:1147/1480 train_time:179277ms step_avg:157.68ms
step:1148/1480 train_time:179446ms step_avg:157.69ms
step:1149/1480 train_time:179617ms step_avg:157.70ms
step:1150/1480 train_time:179786ms step_avg:157.71ms
step:1151/1480 train_time:179958ms step_avg:157.72ms
step:1152/1480 train_time:180130ms step_avg:157.73ms
step:1153/1480 train_time:180303ms step_avg:157.75ms
step:1154/1480 train_time:180470ms step_avg:157.75ms
step:1155/1480 train_time:180642ms step_avg:157.77ms
step:1156/1480 train_time:180821ms step_avg:157.78ms
step:1157/1480 train_time:180991ms step_avg:157.79ms
step:1158/1480 train_time:181158ms step_avg:157.80ms
step:1159/1480 train_time:181325ms step_avg:157.81ms
step:1160/1480 train_time:181492ms step_avg:157.82ms
step:1161/1480 train_time:181663ms step_avg:157.83ms
step:1162/1480 train_time:181833ms step_avg:157.84ms
step:1163/1480 train_time:182002ms step_avg:157.85ms
step:1164/1480 train_time:182170ms step_avg:157.86ms
step:1165/1480 train_time:182336ms step_avg:157.87ms
step:1166/1480 train_time:182504ms step_avg:157.88ms
step:1167/1480 train_time:182672ms step_avg:157.88ms
step:1168/1480 train_time:182840ms step_avg:157.89ms
step:1169/1480 train_time:183008ms step_avg:157.90ms
step:1170/1480 train_time:183178ms step_avg:157.91ms
step:1171/1480 train_time:183344ms step_avg:157.92ms
step:1172/1480 train_time:183510ms step_avg:157.93ms
step:1173/1480 train_time:183682ms step_avg:157.94ms
step:1174/1480 train_time:183864ms step_avg:157.96ms
step:1175/1480 train_time:184035ms step_avg:157.97ms
step:1176/1480 train_time:184206ms step_avg:157.98ms
step:1177/1480 train_time:184383ms step_avg:158.00ms
step:1178/1480 train_time:184552ms step_avg:158.01ms
step:1179/1480 train_time:184718ms step_avg:158.01ms
step:1180/1480 train_time:184899ms step_avg:158.03ms
step:1181/1480 train_time:185069ms step_avg:158.04ms
step:1182/1480 train_time:185238ms step_avg:158.05ms
step:1183/1480 train_time:185408ms step_avg:158.06ms
step:1184/1480 train_time:185579ms step_avg:158.07ms
step:1185/1480 train_time:185752ms step_avg:158.09ms
step:1186/1480 train_time:185924ms step_avg:158.10ms
step:1187/1480 train_time:186106ms step_avg:158.12ms
step:1188/1480 train_time:186271ms step_avg:158.13ms
step:1189/1480 train_time:186442ms step_avg:158.14ms
step:1190/1480 train_time:186610ms step_avg:158.14ms
step:1191/1480 train_time:186781ms step_avg:158.16ms
step:1192/1480 train_time:186947ms step_avg:158.16ms
step:1193/1480 train_time:187113ms step_avg:158.17ms
step:1194/1480 train_time:187283ms step_avg:158.18ms
step:1195/1480 train_time:187457ms step_avg:158.19ms
step:1196/1480 train_time:187639ms step_avg:158.21ms
step:1197/1480 train_time:187809ms step_avg:158.22ms
step:1198/1480 train_time:187993ms step_avg:158.24ms
step:1199/1480 train_time:188162ms step_avg:158.25ms
step:1200/1480 train_time:188331ms step_avg:158.26ms
step:1201/1480 train_time:188499ms step_avg:158.27ms
step:1202/1480 train_time:188681ms step_avg:158.29ms
step:1203/1480 train_time:188858ms step_avg:158.31ms
step:1204/1480 train_time:189033ms step_avg:158.32ms
step:1205/1480 train_time:189201ms step_avg:158.33ms
step:1206/1480 train_time:189369ms step_avg:158.34ms
step:1207/1480 train_time:189539ms step_avg:158.34ms
step:1208/1480 train_time:189706ms step_avg:158.35ms
step:1209/1480 train_time:189879ms step_avg:158.36ms
step:1210/1480 train_time:190056ms step_avg:158.38ms
step:1211/1480 train_time:190231ms step_avg:158.39ms
step:1212/1480 train_time:190403ms step_avg:158.40ms
step:1213/1480 train_time:190576ms step_avg:158.42ms
step:1214/1480 train_time:190754ms step_avg:158.43ms
step:1215/1480 train_time:190928ms step_avg:158.45ms
step:1216/1480 train_time:191099ms step_avg:158.46ms
step:1217/1480 train_time:191274ms step_avg:158.47ms
step:1218/1480 train_time:191444ms step_avg:158.48ms
step:1219/1480 train_time:191624ms step_avg:158.50ms
step:1220/1480 train_time:191795ms step_avg:158.51ms
step:1221/1480 train_time:191962ms step_avg:158.52ms
step:1222/1480 train_time:192129ms step_avg:158.52ms
step:1223/1480 train_time:192300ms step_avg:158.53ms
step:1224/1480 train_time:192477ms step_avg:158.55ms
step:1225/1480 train_time:192650ms step_avg:158.56ms
step:1226/1480 train_time:192823ms step_avg:158.57ms
step:1227/1480 train_time:192997ms step_avg:158.58ms
step:1228/1480 train_time:193166ms step_avg:158.59ms
step:1229/1480 train_time:193339ms step_avg:158.60ms
step:1230/1480 train_time:193519ms step_avg:158.62ms
step:1231/1480 train_time:193696ms step_avg:158.64ms
step:1232/1480 train_time:193870ms step_avg:158.65ms
step:1233/1480 train_time:194039ms step_avg:158.66ms
step:1234/1480 train_time:194210ms step_avg:158.67ms
step:1235/1480 train_time:194385ms step_avg:158.68ms
step:1236/1480 train_time:194553ms step_avg:158.69ms
step:1237/1480 train_time:194724ms step_avg:158.70ms
step:1238/1480 train_time:194909ms step_avg:158.72ms
step:1239/1480 train_time:195080ms step_avg:158.73ms
step:1240/1480 train_time:195252ms step_avg:158.74ms
step:1241/1480 train_time:195424ms step_avg:158.75ms
step:1242/1480 train_time:195593ms step_avg:158.76ms
step:1243/1480 train_time:195765ms step_avg:158.77ms
step:1244/1480 train_time:195932ms step_avg:158.78ms
step:1245/1480 train_time:196102ms step_avg:158.79ms
step:1246/1480 train_time:196271ms step_avg:158.80ms
step:1247/1480 train_time:196440ms step_avg:158.80ms
step:1248/1480 train_time:196609ms step_avg:158.81ms
step:1249/1480 train_time:196778ms step_avg:158.82ms
step:1250/1480 train_time:196947ms step_avg:158.83ms
step:1250/1480 val_loss:3.3384 train_time:197018ms step_avg:158.89ms
step:1251/1480 train_time:197128ms step_avg:158.85ms
step:1252/1480 train_time:197297ms step_avg:158.85ms
step:1253/1480 train_time:197464ms step_avg:158.86ms
step:1254/1480 train_time:197636ms step_avg:158.87ms
step:1255/1480 train_time:197824ms step_avg:158.89ms
step:1256/1480 train_time:197998ms step_avg:158.91ms
step:1257/1480 train_time:198167ms step_avg:158.91ms
step:1258/1480 train_time:198342ms step_avg:158.93ms
step:1259/1480 train_time:198513ms step_avg:158.94ms
step:1260/1480 train_time:198682ms step_avg:158.95ms
step:1261/1480 train_time:198854ms step_avg:158.96ms
step:1262/1480 train_time:199029ms step_avg:158.97ms
step:1263/1480 train_time:199204ms step_avg:158.98ms
step:1264/1480 train_time:199371ms step_avg:158.99ms
step:1265/1480 train_time:199536ms step_avg:158.99ms
step:1266/1480 train_time:199709ms step_avg:159.00ms
step:1267/1480 train_time:199879ms step_avg:159.01ms
step:1268/1480 train_time:200049ms step_avg:159.02ms
step:1269/1480 train_time:200226ms step_avg:159.04ms
step:1270/1480 train_time:200395ms step_avg:159.04ms
step:1271/1480 train_time:200566ms step_avg:159.05ms
step:1272/1480 train_time:200731ms step_avg:159.06ms
step:1273/1480 train_time:200902ms step_avg:159.07ms
step:1274/1480 train_time:201072ms step_avg:159.08ms
step:1275/1480 train_time:201241ms step_avg:159.08ms
step:1276/1480 train_time:201407ms step_avg:159.09ms
step:1277/1480 train_time:201579ms step_avg:159.10ms
step:1278/1480 train_time:201747ms step_avg:159.11ms
step:1279/1480 train_time:201918ms step_avg:159.12ms
step:1280/1480 train_time:202098ms step_avg:159.13ms
step:1281/1480 train_time:202267ms step_avg:159.14ms
step:1282/1480 train_time:202433ms step_avg:159.15ms
step:1283/1480 train_time:202605ms step_avg:159.16ms
step:1284/1480 train_time:202776ms step_avg:159.16ms
step:1285/1480 train_time:202945ms step_avg:159.17ms
step:1286/1480 train_time:203114ms step_avg:159.18ms
step:1287/1480 train_time:203285ms step_avg:159.19ms
step:1288/1480 train_time:203457ms step_avg:159.20ms
step:1289/1480 train_time:203640ms step_avg:159.22ms
step:1290/1480 train_time:203820ms step_avg:159.23ms
step:1291/1480 train_time:203993ms step_avg:159.25ms
step:1292/1480 train_time:204167ms step_avg:159.26ms
step:1293/1480 train_time:204345ms step_avg:159.27ms
step:1294/1480 train_time:204516ms step_avg:159.28ms
step:1295/1480 train_time:204687ms step_avg:159.29ms
step:1296/1480 train_time:204861ms step_avg:159.30ms
step:1297/1480 train_time:205032ms step_avg:159.31ms
step:1298/1480 train_time:205205ms step_avg:159.32ms
step:1299/1480 train_time:205375ms step_avg:159.33ms
step:1300/1480 train_time:205543ms step_avg:159.34ms
step:1301/1480 train_time:205712ms step_avg:159.34ms
step:1302/1480 train_time:205887ms step_avg:159.36ms
step:1303/1480 train_time:206065ms step_avg:159.37ms
step:1304/1480 train_time:206239ms step_avg:159.38ms
step:1305/1480 train_time:206408ms step_avg:159.39ms
step:1306/1480 train_time:206583ms step_avg:159.40ms
step:1307/1480 train_time:206752ms step_avg:159.41ms
step:1308/1480 train_time:206922ms step_avg:159.42ms
step:1309/1480 train_time:207094ms step_avg:159.43ms
step:1310/1480 train_time:207263ms step_avg:159.43ms
step:1311/1480 train_time:207430ms step_avg:159.44ms
step:1312/1480 train_time:207603ms step_avg:159.45ms
step:1313/1480 train_time:207772ms step_avg:159.46ms
step:1314/1480 train_time:207946ms step_avg:159.47ms
step:1315/1480 train_time:208117ms step_avg:159.48ms
step:1316/1480 train_time:208285ms step_avg:159.48ms
step:1317/1480 train_time:208457ms step_avg:159.49ms
step:1318/1480 train_time:208637ms step_avg:159.51ms
step:1319/1480 train_time:208813ms step_avg:159.52ms
step:1320/1480 train_time:208990ms step_avg:159.53ms
step:1321/1480 train_time:209163ms step_avg:159.54ms
step:1322/1480 train_time:209345ms step_avg:159.56ms
step:1323/1480 train_time:209517ms step_avg:159.57ms
step:1324/1480 train_time:209691ms step_avg:159.58ms
step:1325/1480 train_time:209870ms step_avg:159.60ms
step:1326/1480 train_time:210046ms step_avg:159.61ms
step:1327/1480 train_time:210217ms step_avg:159.62ms
step:1328/1480 train_time:210389ms step_avg:159.63ms
step:1329/1480 train_time:210584ms step_avg:159.65ms
step:1330/1480 train_time:210765ms step_avg:159.67ms
step:1331/1480 train_time:210934ms step_avg:159.68ms
step:1332/1480 train_time:211109ms step_avg:159.69ms
step:1333/1480 train_time:211283ms step_avg:159.70ms
step:1334/1480 train_time:211454ms step_avg:159.71ms
step:1335/1480 train_time:211625ms step_avg:159.72ms
step:1336/1480 train_time:211807ms step_avg:159.73ms
step:1337/1480 train_time:211984ms step_avg:159.75ms
step:1338/1480 train_time:212154ms step_avg:159.75ms
step:1339/1480 train_time:212329ms step_avg:159.77ms
step:1340/1480 train_time:212501ms step_avg:159.78ms
step:1341/1480 train_time:212669ms step_avg:159.78ms
step:1342/1480 train_time:212842ms step_avg:159.79ms
step:1343/1480 train_time:213011ms step_avg:159.80ms
step:1344/1480 train_time:213184ms step_avg:159.81ms
step:1345/1480 train_time:213361ms step_avg:159.82ms
step:1346/1480 train_time:213530ms step_avg:159.83ms
step:1347/1480 train_time:213701ms step_avg:159.84ms
step:1348/1480 train_time:213870ms step_avg:159.84ms
step:1349/1480 train_time:214039ms step_avg:159.85ms
step:1350/1480 train_time:214212ms step_avg:159.86ms
step:1351/1480 train_time:214384ms step_avg:159.87ms
step:1352/1480 train_time:214555ms step_avg:159.88ms
step:1353/1480 train_time:214732ms step_avg:159.89ms
step:1354/1480 train_time:214903ms step_avg:159.90ms
step:1355/1480 train_time:215072ms step_avg:159.90ms
step:1356/1480 train_time:215246ms step_avg:159.92ms
step:1357/1480 train_time:215420ms step_avg:159.93ms
step:1358/1480 train_time:215592ms step_avg:159.93ms
step:1359/1480 train_time:215764ms step_avg:159.94ms
step:1360/1480 train_time:215938ms step_avg:159.95ms
step:1361/1480 train_time:216115ms step_avg:159.97ms
step:1362/1480 train_time:216290ms step_avg:159.98ms
step:1363/1480 train_time:216470ms step_avg:159.99ms
step:1364/1480 train_time:216639ms step_avg:160.00ms
step:1365/1480 train_time:216806ms step_avg:160.00ms
step:1366/1480 train_time:216978ms step_avg:160.01ms
step:1367/1480 train_time:217149ms step_avg:160.02ms
step:1368/1480 train_time:217324ms step_avg:160.03ms
step:1369/1480 train_time:217505ms step_avg:160.05ms
step:1370/1480 train_time:217683ms step_avg:160.06ms
step:1371/1480 train_time:217852ms step_avg:160.07ms
step:1372/1480 train_time:218030ms step_avg:160.08ms
step:1373/1480 train_time:218201ms step_avg:160.09ms
step:1374/1480 train_time:218376ms step_avg:160.10ms
step:1375/1480 train_time:218548ms step_avg:160.11ms
step:1375/1480 val_loss:3.3005 train_time:218615ms step_avg:160.16ms
step:1376/1480 train_time:218721ms step_avg:160.12ms
step:1377/1480 train_time:218894ms step_avg:160.13ms
step:1378/1480 train_time:219063ms step_avg:160.13ms
step:1379/1480 train_time:219239ms step_avg:160.15ms
step:1380/1480 train_time:219411ms step_avg:160.15ms
step:1381/1480 train_time:219590ms step_avg:160.17ms
step:1382/1480 train_time:219762ms step_avg:160.18ms
step:1383/1480 train_time:219933ms step_avg:160.18ms
step:1384/1480 train_time:220109ms step_avg:160.20ms
step:1385/1480 train_time:220275ms step_avg:160.20ms
step:1386/1480 train_time:220447ms step_avg:160.21ms
step:1387/1480 train_time:220619ms step_avg:160.22ms
step:1388/1480 train_time:220787ms step_avg:160.22ms
step:1389/1480 train_time:220962ms step_avg:160.23ms
step:1390/1480 train_time:221130ms step_avg:160.24ms
step:1391/1480 train_time:221301ms step_avg:160.25ms
step:1392/1480 train_time:221472ms step_avg:160.25ms
step:1393/1480 train_time:221643ms step_avg:160.26ms
step:1394/1480 train_time:221814ms step_avg:160.27ms
step:1395/1480 train_time:221983ms step_avg:160.28ms
step:1396/1480 train_time:222150ms step_avg:160.28ms
step:1397/1480 train_time:222317ms step_avg:160.29ms
step:1398/1480 train_time:222484ms step_avg:160.29ms
step:1399/1480 train_time:222652ms step_avg:160.30ms
step:1400/1480 train_time:222829ms step_avg:160.31ms
step:1401/1480 train_time:222995ms step_avg:160.31ms
step:1402/1480 train_time:223167ms step_avg:160.32ms
step:1403/1480 train_time:223345ms step_avg:160.33ms
step:1404/1480 train_time:223516ms step_avg:160.34ms
step:1405/1480 train_time:223689ms step_avg:160.35ms
step:1406/1480 train_time:223864ms step_avg:160.36ms
step:1407/1480 train_time:224031ms step_avg:160.37ms
step:1408/1480 train_time:224200ms step_avg:160.37ms
step:1409/1480 train_time:224384ms step_avg:160.39ms
step:1410/1480 train_time:224552ms step_avg:160.39ms
step:1411/1480 train_time:224721ms step_avg:160.40ms
step:1412/1480 train_time:224890ms step_avg:160.41ms
step:1413/1480 train_time:225060ms step_avg:160.41ms
step:1414/1480 train_time:225230ms step_avg:160.42ms
step:1415/1480 train_time:225404ms step_avg:160.43ms
step:1416/1480 train_time:225589ms step_avg:160.45ms
step:1417/1480 train_time:225765ms step_avg:160.46ms
step:1418/1480 train_time:225936ms step_avg:160.47ms
step:1419/1480 train_time:226110ms step_avg:160.48ms
step:1420/1480 train_time:226284ms step_avg:160.49ms
step:1421/1480 train_time:226457ms step_avg:160.49ms
step:1422/1480 train_time:226628ms step_avg:160.50ms
step:1423/1480 train_time:226798ms step_avg:160.51ms
step:1424/1480 train_time:226974ms step_avg:160.52ms
step:1425/1480 train_time:227152ms step_avg:160.53ms
step:1426/1480 train_time:227325ms step_avg:160.54ms
step:1427/1480 train_time:227501ms step_avg:160.55ms
step:1428/1480 train_time:227670ms step_avg:160.56ms
step:1429/1480 train_time:227840ms step_avg:160.56ms
step:1430/1480 train_time:228014ms step_avg:160.57ms
step:1431/1480 train_time:228188ms step_avg:160.58ms
step:1432/1480 train_time:228364ms step_avg:160.59ms
step:1433/1480 train_time:228545ms step_avg:160.61ms
step:1434/1480 train_time:228725ms step_avg:160.62ms
step:1435/1480 train_time:228900ms step_avg:160.63ms
step:1436/1480 train_time:229073ms step_avg:160.64ms
step:1437/1480 train_time:229244ms step_avg:160.65ms
step:1438/1480 train_time:229413ms step_avg:160.65ms
step:1439/1480 train_time:229588ms step_avg:160.66ms
step:1440/1480 train_time:229759ms step_avg:160.67ms
step:1441/1480 train_time:229932ms step_avg:160.68ms
step:1442/1480 train_time:230109ms step_avg:160.69ms
step:1443/1480 train_time:230297ms step_avg:160.71ms
step:1444/1480 train_time:230467ms step_avg:160.72ms
step:1445/1480 train_time:230638ms step_avg:160.72ms
step:1446/1480 train_time:230813ms step_avg:160.73ms
step:1447/1480 train_time:230990ms step_avg:160.74ms
step:1448/1480 train_time:231162ms step_avg:160.75ms
step:1449/1480 train_time:231338ms step_avg:160.76ms
step:1450/1480 train_time:231511ms step_avg:160.77ms
step:1451/1480 train_time:231682ms step_avg:160.78ms
step:1452/1480 train_time:231855ms step_avg:160.79ms
step:1453/1480 train_time:232024ms step_avg:160.79ms
step:1454/1480 train_time:232195ms step_avg:160.80ms
step:1455/1480 train_time:232373ms step_avg:160.81ms
step:1456/1480 train_time:232546ms step_avg:160.82ms
step:1457/1480 train_time:232716ms step_avg:160.83ms
step:1458/1480 train_time:232887ms step_avg:160.83ms
step:1459/1480 train_time:233064ms step_avg:160.84ms
step:1460/1480 train_time:233237ms step_avg:160.85ms
step:1461/1480 train_time:233411ms step_avg:160.86ms
step:1462/1480 train_time:233582ms step_avg:160.87ms
step:1463/1480 train_time:233759ms step_avg:160.88ms
step:1464/1480 train_time:233936ms step_avg:160.89ms
step:1465/1480 train_time:234107ms step_avg:160.90ms
step:1466/1480 train_time:234278ms step_avg:160.91ms
step:1467/1480 train_time:234451ms step_avg:160.91ms
step:1468/1480 train_time:234621ms step_avg:160.92ms
step:1469/1480 train_time:234794ms step_avg:160.93ms
step:1470/1480 train_time:234975ms step_avg:160.94ms
step:1471/1480 train_time:235163ms step_avg:160.96ms
step:1472/1480 train_time:235346ms step_avg:160.98ms
step:1473/1480 train_time:235518ms step_avg:160.98ms
step:1474/1480 train_time:235697ms step_avg:161.00ms
step:1475/1480 train_time:235876ms step_avg:161.01ms
step:1476/1480 train_time:236048ms step_avg:161.01ms
step:1477/1480 train_time:236229ms step_avg:161.03ms
step:1478/1480 train_time:236411ms step_avg:161.04ms
step:1479/1480 train_time:236586ms step_avg:161.05ms
step:1480/1480 train_time:236758ms step_avg:161.06ms
step:1480/1480 val_loss:3.2814 train_time:236830ms step_avg:161.11ms