8a0afde4-391d-4e2d-8df1-e44fe0b80feb.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
from dataclasses import dataclass
from pathlib import Path

import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
# Use of FlexAttention contributed by @KoszarskyB
from torch.nn.attention.flex_attention import BlockMask, flex_attention

# -----------------------------------------------------------------------------
# Muon optimizer

def zeropower_via_svd(G, steps=None):
    U, S, V = G.svd()
    return U @ V.T

@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert len(G.shape) == 2
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    X /= (X.norm() + eps) # ensure top singular value <= 1
    if G.size(0) > G.size(1):
        X = X.T
    for _ in range(steps):
        A = X @ X.T
        B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    if G.size(0) > G.size(1):
        X = X.T
    return X

zeropower_backends = dict(svd=zeropower_via_svd, newtonschulz5=zeropower_via_newtonschulz5)

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven't tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        backend: The chosen backend for the orthogonalization step. (recommended: 'newtonschulz5')
        backend_steps: The number of iteration steps to use in the backend, if it is iterative.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True,
                 backend='newtonschulz5', backend_steps=5):
        self.num_process = int(os.environ['WORLD_SIZE'])
        self.rank = int(os.environ["RANK"])
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, backend=backend, backend_steps=backend_steps)
        params: "list[torch.Tensor]" = list(params)
        assert all(isinstance(p, torch.Tensor) for p in params)
        sizes = {p.numel() for p in params}
        param_groups = [
            {
                "params": [p for p in params if p.numel() == size],
                "update_buffer": [
                    torch.empty(size, device="cuda", dtype=torch.bfloat16)
                    for _ in range(self.num_process)
                ],
            }
            for size in sizes
        ]
        super().__init__(param_groups, defaults)

    def step(self):
        for group in self.param_groups:
            lr: float = group["lr"]
            momentum: float = group["momentum"]
            nesterov: bool = group["nesterov"]
            zeropower_backend = zeropower_backends[group["backend"]]
            backend_steps: int = group["backend_steps"]
            update_buffers: "list[torch.Tensor]" = group["update_buffer"]
            # generate weight updates in distributed fashion
            params: "list[torch.Tensor]" = group["params"]
            assert len(params) % self.num_process == 0
            handle = None
            params_world = None
            def update_prev():
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffers):
                    p_world.data.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.num_process]:
                p = params[base_i + self.rank]
                g = p.grad
                assert g is not None
                state = self.state[p] 
                if "momentum_buffer" not in state:
                    state["momentum_buffer"] = torch.zeros_like(g)
                buf: torch.Tensor = state["momentum_buffer"]
                buf.lerp_(g, 1 - momentum)
                g = g.lerp_(buf, momentum) if nesterov else buf
                g = zeropower_backend(g, steps=backend_steps).flatten()
                update_prev()
                handle = dist.all_gather(update_buffers, g, async_op=True)
                params_world = params[base_i : base_i + self.num_process]
            update_prev()


# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):

    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=False)

    def forward(self, x):
        return F.linear(x, self.weight.to(x.dtype))

class Rotary(torch.nn.Module):

    def __init__(self, dim, base=10000):
        super().__init__()
        self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None

    def forward(self, x):
        seq_len = x.shape[1]
        if seq_len != self.seq_len_cached:
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.outer(t, self.inv_freq)
            self.seq_len_cached = seq_len
            self.cos_cached = freqs.cos()
            self.sin_cached = freqs.sin()
        cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
        # apply_rotary_emb(x, cos, sin)
        x1, x2 = x.chunk(2, dim=3)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x)

class CausalSelfAttention(nn.Module):

    def __init__(self, dim, n_head):
        super().__init__()
        assert dim % n_head == 0
        self.n_head = n_head
        self.c_q = CastedLinear(dim, dim)
        self.c_k = CastedLinear(dim, dim)
        self.c_v = CastedLinear(dim, dim)
        # value residual lambda
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5])) # @Grad62304977
        # rotary embeddings
        self.rotary = Rotary(dim // n_head) # dim // n_head = head_dim
        # output projection
        self.c_proj = CastedLinear(dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor, vi: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q: torch.Tensor = self.c_q(x).view(B, T, self.n_head, -1)
        k: torch.Tensor = self.c_k(x).view(B, T, self.n_head, -1)
        v: torch.Tensor = self.c_v(x).view(B, T, self.n_head, -1)
        v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @Grad62304977
        q, k = norm(q), norm(k) # QK norm suggested by @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask)
        y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):

    def __init__(self, dim: int):
        super().__init__()
        self.c_fc   = CastedLinear(dim, 4 * dim)
        self.c_proj = CastedLinear(4 * dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):

    def __init__(self, config):
        super().__init__()
        self.attn = CausalSelfAttention(config.n_embd, config.n_head)
        self.mlp = MLP(config.n_embd)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x: torch.Tensor, vi: torch.Tensor, x0: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        x = x + self.attn(norm(x), vi, block_mask)
        x = x + self.mlp(norm(x))
        return x

# -----------------------------------------------------------------------------
# The main GPT-2 model

@dataclass
class GPTConfig:
    vocab_size : int = 50304
    n_layer : int = 12
    n_head : int = 6 # head dim 128 suggested by @Grad62304977
    n_embd : int = 768
    lm_head_softcap : int = 30

class GPT(nn.Module):

    def __init__(self, config: GPTConfig):
        super().__init__()
        self.n_layer = config.n_layer
        self.lm_head_softcap = config.lm_head_softcap

        # U-net design by @brendanh0gan
        self.num_encoder_layers = config.n_layer // 2 # Half of the layers for encoder
        self.num_decoder_layers = config.n_layer - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))

        self.transformer = nn.ModuleDict(dict(
            wte = nn.Embedding(config.vocab_size, config.n_embd),
            # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
            # U-net structure on token value embeddings by @leloykun
            vte = nn.Embedding(config.vocab_size, config.n_embd*self.num_encoder_layers),
            h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
        ))
        self.lm_head = CastedLinear(config.n_embd, config.vocab_size)
        self.lm_head.weight.data.zero_() # @Grad62304977

    def forward(self, idx: torch.Tensor, target: torch.Tensor, sliding_window: torch.Tensor) -> torch.Tensor:
        BLOCK_SIZE = 128
        assert idx.ndim == 1
        docs = (idx == 50256).cumsum(0)
        docs_low = docs.reshape(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.reshape(-1, BLOCK_SIZE)[:, -1].contiguous()
        def document_sliding_window_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            window_mask = q_idx - kv_idx < sliding_window
            return causal_mask & document_mask & window_mask

        S = len(idx)
        def create_sliding_window_causal_mask(S: int, sliding_window: torch.Tensor):
            kv_idx = block_idx = torch.arange(S // BLOCK_SIZE, dtype=torch.int32, device="cuda")
            q_idx = block_idx[:, None]
            causal_mask = q_idx >= kv_idx
            document_mask = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
            window_mask = q_idx - kv_idx < ((sliding_window + BLOCK_SIZE - 1) // BLOCK_SIZE)
            dense_mask = causal_mask & document_mask & window_mask
            dense_mask = dense_mask.to(torch.int32)
            num_blocks = dense_mask.sum(dim=-1).to(torch.int32)
            indices = torch.argsort(dense_mask, dim=-1, descending=True, stable=True).to(torch.int32)
            num_blocks = num_blocks[None, None, :].contiguous()
            indices = indices[None, None, :].contiguous()
            return BlockMask.from_kv_blocks(num_blocks, indices, BLOCK_SIZE=BLOCK_SIZE, mask_mod=document_sliding_window_causal)
        block_mask = create_sliding_window_causal_mask(S, sliding_window)

        # forward the GPT model itself
        x = self.transformer.wte(idx[None]) # token embeddings of shape (b, t, n_embd)
        x = norm(x) # @Grad62304977
        x0 = x
        vi = self.transformer.vte(idx[None]).chunk(self.num_encoder_layers, dim=-1)

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        for i in range(self.num_encoder_layers):
            x = self.transformer.h[i](x, vi[i], x0, block_mask)
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            # U-net structure on token value embeddings by @leloykun
            x = self.transformer.h[self.num_encoder_layers + i](x, vi[self.num_encoder_layers-1-i], x0, block_mask)

        x = norm(x)
        logits = self.lm_head(x)
        logits = self.lm_head_softcap * torch.tanh(logits / self.lm_head_softcap) # @Grad62304977
        logits = logits.float()
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), target.view(-1))
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _peek_data_shard(file: Path):
    # only reads the header, returns header data
    # header is 256 int32
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    return int(header[2]) # number of tokens (claimed)

def _load_data_shard(file: Path, ntok: int):
    with file.open("rb") as f:
        tokens = torch.empty(ntok, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy())
        assert nbytes == 2 * ntok, "number of tokens read does not match header?"
    return tokens

class DistributedDataLoader:
    def __init__(self, filename_pattern, T, process_rank, num_processes):
        self.process_rank = process_rank
        self.num_processes = num_processes
        self.T = T

        # glob files that match the pattern
        self.files = sorted(Path.cwd().glob(filename_pattern))
        assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"

        # load and validate all data shards, count number of tokens in total
        self.ntoks = [_peek_data_shard(file) for file in self.files]
        assert min(self.ntoks) >= num_processes * T + 1
        self.ntok_total = sum(self.ntoks)

        self.reset()

    def reset(self):
        self.current_shard = -1
        self.advance()

    def advance(self): # advance to next data shard
        self.current_shard = (self.current_shard + 1) % len(self.files)
        self.current_position = self.process_rank * self.T
        self.tokens = _load_data_shard(self.files[self.current_shard], self.ntoks[self.current_shard])

    def next_batch(self):
        batch_size = self.T * self.num_processes
        buf = self.tokens[self.current_position:self.current_position+self.T+1]
        # host side async is sufficient;
        # no performance improvement was observed when introducing a separate stream.
        x = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
        y = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
        # advance current position and load next shard if necessary
        self.current_position += batch_size
        if self.current_position + batch_size + 1 >= len(self.tokens):
            self.advance()
        return x, y

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data hyperparams
    input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
    input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
    # optimization hyperparams
    batch_size : int = 8 # batch size, in sequences, across all devices
    sequence_length : int = 64*1024 # sequence length, in tokens
    num_iterations : int = 1480 # number of iterations to run
    warmup_iters : int = 0
    cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
    weight_decay : float = 0
    # evaluation and logging hyperparams
    val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
    val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()

# set up DDP (distributed data parallel). torchrun sets this env variable
assert torch.cuda.is_available()
dist.init_process_group(backend='nccl')
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
device = f'cuda:{ddp_local_rank}'
torch.cuda.set_device(device)
print(f"using device: {device}")
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = str(uuid.uuid4())
    logdir = 'logs/%s/' % run_id
    # os.makedirs(logdir, exist_ok=True)
    logfile = 'logs/%s.txt' % run_id
    # create the log file
    with open(logfile, "w") as f:
        # begin the log by printing this file (the Python code)
        f.write(code)
        f.write('='*100 + '\n')
def print0(s, logonly=False):
    if master_process:
        with open(logfile, "a") as f:
            if not logonly:
                print(s)
            f.write(s+'\n')
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)

# convenience variables
T = args.sequence_length
# calculate the number of steps to take in the val loop.
assert args.val_tokens % (T * ddp_world_size) == 0
val_steps = args.val_tokens // (T * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size
assert train_accumulation_steps == 1

# load tokens
train_loader = DistributedDataLoader(args.input_bin, T, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, T, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.ntok_total} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.ntok_total} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
x, y = train_loader.next_batch()

# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, n_layer=12, n_head=6, n_embd=768))
model = model.cuda().bfloat16()
for m in model.modules():
    if isinstance(m, CastedLinear):
        m.float()
if hasattr(config, "coordinate_descent_tuning"):
    config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank])
raw_model = model.module # always contains the "raw" unwrapped model

# init the optimizer(s)
optimizer1 = torch.optim.Adam([raw_model.transformer.wte.weight, raw_model.transformer.vte.weight], lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.transformer.h.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
    assert it <= args.num_iterations
    # 1) linear warmup for warmup_iters steps
    if it < args.warmup_iters:
        return (it+1) / args.warmup_iters
    # 2) constant lr for a while
    elif it < args.num_iterations - args.cooldown_iters:
        return 1.0
    # 3) linear cooldown
    else:
        decay_ratio = (args.num_iterations - it) / args.cooldown_iters
        return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]

sliding_window_size = torch.tensor(64, dtype=torch.int32, device="cuda")
sw_size_prev = 64
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
    last_step = (step == args.num_iterations)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Set the sliding window size for the current step, in chunks of 64. By @fernbear.bsky.social
    sw_size =  64 * int((64 + (1792 - 64) * step / args.num_iterations) // 64)
    if sw_size != sw_size_prev:
        sliding_window_size.copy_(sw_size, non_blocking=True)
        sw_size_prev = sw_size

    # once in a while evaluate the validation dataset
    if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # run validation batches
        model.eval()
        val_loader.reset()
        val_loss = 0.0
        for _ in range(val_steps):
            with torch.no_grad():
                x_val, y_val = val_loader.next_batch()
                val_loss += model(x_val, y_val, sliding_window=sliding_window_size)
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        val_loss /= val_steps
        # log val loss to console and to logfile
        print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # save the state of the training process
        log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
        # torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    # bit confusing: we want to make sure to eval on 0th iteration
    # but also after the very last iteration. so we loop for step <= num_iterations
    # instead of just < num_iterations (one extra due to <=), only to do
    # the validation/sampling one last time, and then we break right here as we're done.
    if last_step:
        break

    # --------------- TRAINING SECTION BEGIN -----------------
    model.train()
    loss = model(x, y, sliding_window=sliding_window_size)
    loss.backward()
    del loss
    # advance the dataset for the next batch
    x, y = train_loader.next_batch()
    # momentum warmup for Muon
    frac = min(step/300, 1)
    for group in optimizer3.param_groups:
        group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # --------------- TRAINING SECTION END -------------------
    # everything that follows now is just diagnostics, prints, logging, etc.
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")

if master_process:
    print(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")

# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()
====================================================================================================
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Sun Dec  8 09:37:43 2024       
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.6     |
|-----------------------------------------+----------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
|                                         |                      |               MIG M. |
|=========================================+======================+======================|
|   0  NVIDIA H100 80GB HBM3          On  | 00000000:65:02.0 Off |                    0 |
| N/A   37C    P0              74W / 700W |      7MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  | 00000000:67:02.0 Off |                    0 |
| N/A   46C    P0             125W / 700W |    533MiB / 81559MiB |      2%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  | 00000000:69:02.0 Off |                    0 |
| N/A   46C    P0             123W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  | 00000000:6B:02.0 Off |                    0 |
| N/A   39C    P0             118W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  | 00000000:6F:02.0 Off |                    0 |
| N/A   39C    P0             117W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  | 00000000:71:02.0 Off |                    0 |
| N/A   46C    P0             122W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  | 00000000:73:02.0 Off |                    0 |
| N/A   46C    P0             127W / 700W |    533MiB / 81559MiB |      1%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  | 00000000:75:02.0 Off |                    0 |
| N/A   38C    P0             120W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
                                                                                         
+---------------------------------------------------------------------------------------+
| Processes:                                                                            |
|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
|        ID   ID                                                             Usage      |
|=======================================================================================|
+---------------------------------------------------------------------------------------+

====================================================================================================
Training DataLoader: total number of tokens: 3200000000 across 32 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:23374ms step_avg:nanms
step:2/1480 train_time:23508ms step_avg:nanms
step:3/1480 train_time:23648ms step_avg:nanms
step:4/1480 train_time:23790ms step_avg:nanms
step:5/1480 train_time:23933ms step_avg:nanms
step:6/1480 train_time:24074ms step_avg:nanms
step:7/1480 train_time:24215ms step_avg:nanms
step:8/1480 train_time:24356ms step_avg:nanms
step:9/1480 train_time:24500ms step_avg:nanms
step:10/1480 train_time:24645ms step_avg:nanms
step:11/1480 train_time:143ms step_avg:nanms
step:12/1480 train_time:286ms step_avg:nanms
step:13/1480 train_time:428ms step_avg:142.55ms
step:14/1480 train_time:569ms step_avg:142.31ms
step:15/1480 train_time:712ms step_avg:142.32ms
step:16/1480 train_time:853ms step_avg:142.18ms
step:17/1480 train_time:997ms step_avg:142.47ms
step:18/1480 train_time:1142ms step_avg:142.73ms
step:19/1480 train_time:1287ms step_avg:143.01ms
step:20/1480 train_time:1431ms step_avg:143.10ms
step:21/1480 train_time:1573ms step_avg:142.99ms
step:22/1480 train_time:1715ms step_avg:142.88ms
step:23/1480 train_time:1856ms step_avg:142.77ms
step:24/1480 train_time:1998ms step_avg:142.72ms
step:25/1480 train_time:2140ms step_avg:142.68ms
step:26/1480 train_time:2284ms step_avg:142.75ms
step:27/1480 train_time:2429ms step_avg:142.86ms
step:28/1480 train_time:2572ms step_avg:142.90ms
step:29/1480 train_time:2713ms step_avg:142.80ms
step:30/1480 train_time:2854ms step_avg:142.69ms
step:31/1480 train_time:2995ms step_avg:142.62ms
step:32/1480 train_time:3137ms step_avg:142.58ms
step:33/1480 train_time:3281ms step_avg:142.64ms
step:34/1480 train_time:3424ms step_avg:142.66ms
step:35/1480 train_time:3567ms step_avg:142.68ms
step:36/1480 train_time:3709ms step_avg:142.67ms
step:37/1480 train_time:3850ms step_avg:142.60ms
step:38/1480 train_time:3992ms step_avg:142.57ms
step:39/1480 train_time:4133ms step_avg:142.53ms
step:40/1480 train_time:4275ms step_avg:142.50ms
step:41/1480 train_time:4417ms step_avg:142.48ms
step:42/1480 train_time:4560ms step_avg:142.50ms
step:43/1480 train_time:4704ms step_avg:142.54ms
step:44/1480 train_time:4847ms step_avg:142.57ms
step:45/1480 train_time:4991ms step_avg:142.59ms
step:46/1480 train_time:5133ms step_avg:142.57ms
step:47/1480 train_time:5274ms step_avg:142.55ms
step:48/1480 train_time:5416ms step_avg:142.52ms
step:49/1480 train_time:5560ms step_avg:142.57ms
step:50/1480 train_time:5704ms step_avg:142.61ms
step:51/1480 train_time:5847ms step_avg:142.61ms
step:52/1480 train_time:5989ms step_avg:142.60ms
step:53/1480 train_time:6132ms step_avg:142.61ms
step:54/1480 train_time:6274ms step_avg:142.60ms
step:55/1480 train_time:6415ms step_avg:142.56ms
step:56/1480 train_time:6558ms step_avg:142.56ms
step:57/1480 train_time:6701ms step_avg:142.57ms
step:58/1480 train_time:6844ms step_avg:142.59ms
step:59/1480 train_time:6988ms step_avg:142.60ms
step:60/1480 train_time:7130ms step_avg:142.59ms
step:61/1480 train_time:7273ms step_avg:142.61ms
step:62/1480 train_time:7414ms step_avg:142.58ms
step:63/1480 train_time:7556ms step_avg:142.56ms
step:64/1480 train_time:7700ms step_avg:142.59ms
step:65/1480 train_time:7844ms step_avg:142.61ms
step:66/1480 train_time:7987ms step_avg:142.62ms
step:67/1480 train_time:8129ms step_avg:142.61ms
step:68/1480 train_time:8271ms step_avg:142.60ms
step:69/1480 train_time:8413ms step_avg:142.59ms
step:70/1480 train_time:8554ms step_avg:142.57ms
step:71/1480 train_time:8699ms step_avg:142.61ms
step:72/1480 train_time:8842ms step_avg:142.62ms
step:73/1480 train_time:8986ms step_avg:142.63ms
step:74/1480 train_time:9129ms step_avg:142.64ms
step:75/1480 train_time:9270ms step_avg:142.62ms
step:76/1480 train_time:9412ms step_avg:142.60ms
step:77/1480 train_time:9552ms step_avg:142.57ms
step:78/1480 train_time:9696ms step_avg:142.59ms
step:79/1480 train_time:9840ms step_avg:142.61ms
step:80/1480 train_time:9984ms step_avg:142.62ms
step:81/1480 train_time:10126ms step_avg:142.62ms
step:82/1480 train_time:10268ms step_avg:142.61ms
step:83/1480 train_time:10411ms step_avg:142.61ms
step:84/1480 train_time:10551ms step_avg:142.58ms
step:85/1480 train_time:10693ms step_avg:142.58ms
step:86/1480 train_time:10835ms step_avg:142.57ms
step:87/1480 train_time:10977ms step_avg:142.56ms
step:88/1480 train_time:11119ms step_avg:142.55ms
step:89/1480 train_time:11262ms step_avg:142.56ms
step:90/1480 train_time:11406ms step_avg:142.57ms
step:91/1480 train_time:11549ms step_avg:142.57ms
step:92/1480 train_time:11691ms step_avg:142.57ms
step:93/1480 train_time:11834ms step_avg:142.58ms
step:94/1480 train_time:11975ms step_avg:142.56ms
step:95/1480 train_time:12116ms step_avg:142.54ms
step:96/1480 train_time:12258ms step_avg:142.54ms
step:97/1480 train_time:12401ms step_avg:142.54ms
step:98/1480 train_time:12544ms step_avg:142.54ms
step:99/1480 train_time:12687ms step_avg:142.55ms
step:100/1480 train_time:12830ms step_avg:142.56ms
step:101/1480 train_time:12973ms step_avg:142.56ms
step:102/1480 train_time:13114ms step_avg:142.54ms
step:103/1480 train_time:13254ms step_avg:142.52ms
step:104/1480 train_time:13395ms step_avg:142.50ms
step:105/1480 train_time:13536ms step_avg:142.49ms
step:106/1480 train_time:13678ms step_avg:142.48ms
step:107/1480 train_time:13820ms step_avg:142.47ms
step:108/1480 train_time:13962ms step_avg:142.47ms
step:109/1480 train_time:14105ms step_avg:142.47ms
step:110/1480 train_time:14247ms step_avg:142.47ms
step:111/1480 train_time:14393ms step_avg:142.50ms
step:112/1480 train_time:14538ms step_avg:142.53ms
step:113/1480 train_time:14685ms step_avg:142.58ms
step:114/1480 train_time:14833ms step_avg:142.62ms
step:115/1480 train_time:14979ms step_avg:142.66ms
step:116/1480 train_time:15127ms step_avg:142.71ms
step:117/1480 train_time:15275ms step_avg:142.75ms
step:118/1480 train_time:15421ms step_avg:142.79ms
step:119/1480 train_time:15569ms step_avg:142.84ms
step:120/1480 train_time:15715ms step_avg:142.86ms
step:121/1480 train_time:15860ms step_avg:142.89ms
step:122/1480 train_time:16008ms step_avg:142.93ms
step:123/1480 train_time:16153ms step_avg:142.95ms
step:124/1480 train_time:16300ms step_avg:142.98ms
step:125/1480 train_time:16447ms step_avg:143.02ms
step:125/1480 val_loss:4.4230 train_time:16505ms step_avg:143.52ms
step:126/1480 train_time:16600ms step_avg:143.11ms
step:127/1480 train_time:16748ms step_avg:143.15ms
step:128/1480 train_time:16896ms step_avg:143.18ms
step:129/1480 train_time:17042ms step_avg:143.21ms
step:130/1480 train_time:17186ms step_avg:143.22ms
step:131/1480 train_time:17332ms step_avg:143.24ms
step:132/1480 train_time:17479ms step_avg:143.27ms
step:133/1480 train_time:17627ms step_avg:143.31ms
step:134/1480 train_time:17774ms step_avg:143.34ms
step:135/1480 train_time:17922ms step_avg:143.38ms
step:136/1480 train_time:18068ms step_avg:143.40ms
step:137/1480 train_time:18214ms step_avg:143.41ms
step:138/1480 train_time:18360ms step_avg:143.44ms
step:139/1480 train_time:18506ms step_avg:143.46ms
step:140/1480 train_time:18653ms step_avg:143.48ms
step:141/1480 train_time:18802ms step_avg:143.53ms
step:142/1480 train_time:18948ms step_avg:143.55ms
step:143/1480 train_time:19096ms step_avg:143.58ms
step:144/1480 train_time:19243ms step_avg:143.61ms
step:145/1480 train_time:19388ms step_avg:143.62ms
step:146/1480 train_time:19535ms step_avg:143.64ms
step:147/1480 train_time:19683ms step_avg:143.67ms
step:148/1480 train_time:19830ms step_avg:143.70ms
step:149/1480 train_time:19979ms step_avg:143.73ms
step:150/1480 train_time:20125ms step_avg:143.75ms
step:151/1480 train_time:20272ms step_avg:143.77ms
step:152/1480 train_time:20420ms step_avg:143.80ms
step:153/1480 train_time:20565ms step_avg:143.81ms
step:154/1480 train_time:20711ms step_avg:143.83ms
step:155/1480 train_time:20859ms step_avg:143.86ms
step:156/1480 train_time:21006ms step_avg:143.87ms
step:157/1480 train_time:21151ms step_avg:143.89ms
step:158/1480 train_time:21300ms step_avg:143.92ms
step:159/1480 train_time:21446ms step_avg:143.94ms
step:160/1480 train_time:21592ms step_avg:143.95ms
step:161/1480 train_time:21741ms step_avg:143.98ms
step:162/1480 train_time:21887ms step_avg:143.99ms
step:163/1480 train_time:22035ms step_avg:144.02ms
step:164/1480 train_time:22182ms step_avg:144.04ms
step:165/1480 train_time:22329ms step_avg:144.06ms
step:166/1480 train_time:22478ms step_avg:144.09ms
step:167/1480 train_time:22625ms step_avg:144.11ms
step:168/1480 train_time:22771ms step_avg:144.12ms
step:169/1480 train_time:22919ms step_avg:144.15ms
step:170/1480 train_time:23066ms step_avg:144.16ms
step:171/1480 train_time:23214ms step_avg:144.18ms
step:172/1480 train_time:23361ms step_avg:144.20ms
step:173/1480 train_time:23507ms step_avg:144.21ms
step:174/1480 train_time:23653ms step_avg:144.23ms
step:175/1480 train_time:23802ms step_avg:144.25ms
step:176/1480 train_time:23948ms step_avg:144.26ms
step:177/1480 train_time:24094ms step_avg:144.28ms
step:178/1480 train_time:24242ms step_avg:144.30ms
step:179/1480 train_time:24388ms step_avg:144.31ms
step:180/1480 train_time:24536ms step_avg:144.33ms
step:181/1480 train_time:24682ms step_avg:144.34ms
step:182/1480 train_time:24828ms step_avg:144.35ms
step:183/1480 train_time:24976ms step_avg:144.37ms
step:184/1480 train_time:25124ms step_avg:144.39ms
step:185/1480 train_time:25269ms step_avg:144.40ms
step:186/1480 train_time:25417ms step_avg:144.41ms
step:187/1480 train_time:25563ms step_avg:144.43ms
step:188/1480 train_time:25709ms step_avg:144.43ms
step:189/1480 train_time:25856ms step_avg:144.45ms
step:190/1480 train_time:26004ms step_avg:144.47ms
step:191/1480 train_time:26150ms step_avg:144.48ms
step:192/1480 train_time:26300ms step_avg:144.50ms
step:193/1480 train_time:26447ms step_avg:144.52ms
step:194/1480 train_time:26595ms step_avg:144.54ms
step:195/1480 train_time:26742ms step_avg:144.55ms
step:196/1480 train_time:26887ms step_avg:144.56ms
step:197/1480 train_time:27036ms step_avg:144.58ms
step:198/1480 train_time:27182ms step_avg:144.59ms
step:199/1480 train_time:27329ms step_avg:144.60ms
step:200/1480 train_time:27477ms step_avg:144.62ms
step:201/1480 train_time:27626ms step_avg:144.64ms
step:202/1480 train_time:27771ms step_avg:144.64ms
step:203/1480 train_time:27919ms step_avg:144.66ms
step:204/1480 train_time:28065ms step_avg:144.67ms
step:205/1480 train_time:28211ms step_avg:144.67ms
step:206/1480 train_time:28358ms step_avg:144.68ms
step:207/1480 train_time:28506ms step_avg:144.70ms
step:208/1480 train_time:28652ms step_avg:144.71ms
step:209/1480 train_time:28799ms step_avg:144.72ms
step:210/1480 train_time:28946ms step_avg:144.73ms
step:211/1480 train_time:29092ms step_avg:144.73ms
step:212/1480 train_time:29239ms step_avg:144.75ms
step:213/1480 train_time:29385ms step_avg:144.75ms
step:214/1480 train_time:29532ms step_avg:144.76ms
step:215/1480 train_time:29679ms step_avg:144.78ms
step:216/1480 train_time:29826ms step_avg:144.79ms
step:217/1480 train_time:29973ms step_avg:144.80ms
step:218/1480 train_time:30120ms step_avg:144.81ms
step:219/1480 train_time:30266ms step_avg:144.81ms
step:220/1480 train_time:30411ms step_avg:144.81ms
step:221/1480 train_time:30560ms step_avg:144.83ms
step:222/1480 train_time:30711ms step_avg:144.86ms
step:223/1480 train_time:30862ms step_avg:144.89ms
step:224/1480 train_time:31011ms step_avg:144.91ms
step:225/1480 train_time:31162ms step_avg:144.94ms
step:226/1480 train_time:31312ms step_avg:144.96ms
step:227/1480 train_time:31462ms step_avg:144.99ms
step:228/1480 train_time:31612ms step_avg:145.01ms
step:229/1480 train_time:31763ms step_avg:145.04ms
step:230/1480 train_time:31913ms step_avg:145.06ms
step:231/1480 train_time:32064ms step_avg:145.08ms
step:232/1480 train_time:32215ms step_avg:145.11ms
step:233/1480 train_time:32366ms step_avg:145.14ms
step:234/1480 train_time:32516ms step_avg:145.16ms
step:235/1480 train_time:32666ms step_avg:145.18ms
step:236/1480 train_time:32816ms step_avg:145.21ms
step:237/1480 train_time:32967ms step_avg:145.23ms
step:238/1480 train_time:33118ms step_avg:145.26ms
step:239/1480 train_time:33269ms step_avg:145.28ms
step:240/1480 train_time:33420ms step_avg:145.31ms
step:241/1480 train_time:33570ms step_avg:145.33ms
step:242/1480 train_time:33721ms step_avg:145.35ms
step:243/1480 train_time:33872ms step_avg:145.37ms
step:244/1480 train_time:34024ms step_avg:145.40ms
step:245/1480 train_time:34173ms step_avg:145.42ms
step:246/1480 train_time:34324ms step_avg:145.44ms
step:247/1480 train_time:34474ms step_avg:145.46ms
step:248/1480 train_time:34625ms step_avg:145.48ms
step:249/1480 train_time:34775ms step_avg:145.50ms
step:250/1480 train_time:34926ms step_avg:145.52ms
step:250/1480 val_loss:3.9959 train_time:34985ms step_avg:145.77ms
step:251/1480 train_time:35081ms step_avg:145.56ms
step:252/1480 train_time:35234ms step_avg:145.59ms
step:253/1480 train_time:35384ms step_avg:145.61ms
step:254/1480 train_time:35533ms step_avg:145.63ms
step:255/1480 train_time:35682ms step_avg:145.64ms
step:256/1480 train_time:35833ms step_avg:145.66ms
step:257/1480 train_time:35982ms step_avg:145.68ms
step:258/1480 train_time:36134ms step_avg:145.70ms
step:259/1480 train_time:36285ms step_avg:145.72ms
step:260/1480 train_time:36436ms step_avg:145.74ms
step:261/1480 train_time:36585ms step_avg:145.76ms
step:262/1480 train_time:36734ms step_avg:145.77ms
step:263/1480 train_time:36883ms step_avg:145.78ms
step:264/1480 train_time:37033ms step_avg:145.80ms
step:265/1480 train_time:37183ms step_avg:145.82ms
step:266/1480 train_time:37336ms step_avg:145.84ms
step:267/1480 train_time:37486ms step_avg:145.86ms
step:268/1480 train_time:37637ms step_avg:145.88ms
step:269/1480 train_time:37785ms step_avg:145.89ms
step:270/1480 train_time:37936ms step_avg:145.91ms
step:271/1480 train_time:38085ms step_avg:145.92ms
step:272/1480 train_time:38235ms step_avg:145.94ms
step:273/1480 train_time:38385ms step_avg:145.95ms
step:274/1480 train_time:38536ms step_avg:145.97ms
step:275/1480 train_time:38686ms step_avg:145.99ms
step:276/1480 train_time:38837ms step_avg:146.00ms
step:277/1480 train_time:38986ms step_avg:146.01ms
step:278/1480 train_time:39136ms step_avg:146.03ms
step:279/1480 train_time:39285ms step_avg:146.04ms
step:280/1480 train_time:39438ms step_avg:146.06ms
step:281/1480 train_time:39588ms step_avg:146.08ms
step:282/1480 train_time:39740ms step_avg:146.10ms
step:283/1480 train_time:39890ms step_avg:146.12ms
step:284/1480 train_time:40040ms step_avg:146.13ms
step:285/1480 train_time:40190ms step_avg:146.14ms
step:286/1480 train_time:40341ms step_avg:146.16ms
step:287/1480 train_time:40491ms step_avg:146.18ms
step:288/1480 train_time:40641ms step_avg:146.19ms
step:289/1480 train_time:40792ms step_avg:146.21ms
step:290/1480 train_time:40942ms step_avg:146.22ms
step:291/1480 train_time:41093ms step_avg:146.24ms
step:292/1480 train_time:41243ms step_avg:146.25ms
step:293/1480 train_time:41394ms step_avg:146.27ms
step:294/1480 train_time:41545ms step_avg:146.28ms
step:295/1480 train_time:41695ms step_avg:146.30ms
step:296/1480 train_time:41846ms step_avg:146.32ms
step:297/1480 train_time:41997ms step_avg:146.33ms
step:298/1480 train_time:42147ms step_avg:146.35ms
step:299/1480 train_time:42298ms step_avg:146.36ms
step:300/1480 train_time:42449ms step_avg:146.38ms
step:301/1480 train_time:42599ms step_avg:146.39ms
step:302/1480 train_time:42750ms step_avg:146.40ms
step:303/1480 train_time:42900ms step_avg:146.42ms
step:304/1480 train_time:43051ms step_avg:146.43ms
step:305/1480 train_time:43201ms step_avg:146.44ms
step:306/1480 train_time:43353ms step_avg:146.46ms
step:307/1480 train_time:43503ms step_avg:146.47ms
step:308/1480 train_time:43654ms step_avg:146.49ms
step:309/1480 train_time:43804ms step_avg:146.50ms
step:310/1480 train_time:43956ms step_avg:146.52ms
step:311/1480 train_time:44106ms step_avg:146.53ms
step:312/1480 train_time:44258ms step_avg:146.55ms
step:313/1480 train_time:44409ms step_avg:146.56ms
step:314/1480 train_time:44559ms step_avg:146.58ms
step:315/1480 train_time:44710ms step_avg:146.59ms
step:316/1480 train_time:44860ms step_avg:146.60ms
step:317/1480 train_time:45011ms step_avg:146.62ms
step:318/1480 train_time:45162ms step_avg:146.63ms
step:319/1480 train_time:45312ms step_avg:146.64ms
step:320/1480 train_time:45462ms step_avg:146.65ms
step:321/1480 train_time:45613ms step_avg:146.67ms
step:322/1480 train_time:45763ms step_avg:146.68ms
step:323/1480 train_time:45915ms step_avg:146.69ms
step:324/1480 train_time:46065ms step_avg:146.70ms
step:325/1480 train_time:46215ms step_avg:146.72ms
step:326/1480 train_time:46365ms step_avg:146.72ms
step:327/1480 train_time:46517ms step_avg:146.74ms
step:328/1480 train_time:46668ms step_avg:146.75ms
step:329/1480 train_time:46819ms step_avg:146.77ms
step:330/1480 train_time:46970ms step_avg:146.78ms
step:331/1480 train_time:47124ms step_avg:146.80ms
step:332/1480 train_time:47277ms step_avg:146.82ms
step:333/1480 train_time:47432ms step_avg:146.85ms
step:334/1480 train_time:47585ms step_avg:146.87ms
step:335/1480 train_time:47738ms step_avg:146.89ms
step:336/1480 train_time:47893ms step_avg:146.91ms
step:337/1480 train_time:48048ms step_avg:146.94ms
step:338/1480 train_time:48201ms step_avg:146.95ms
step:339/1480 train_time:48355ms step_avg:146.98ms
step:340/1480 train_time:48509ms step_avg:147.00ms
step:341/1480 train_time:48662ms step_avg:147.02ms
step:342/1480 train_time:48816ms step_avg:147.04ms
step:343/1480 train_time:48971ms step_avg:147.06ms
step:344/1480 train_time:49125ms step_avg:147.08ms
step:345/1480 train_time:49279ms step_avg:147.10ms
step:346/1480 train_time:49433ms step_avg:147.12ms
step:347/1480 train_time:49586ms step_avg:147.14ms
step:348/1480 train_time:49739ms step_avg:147.16ms
step:349/1480 train_time:49893ms step_avg:147.18ms
step:350/1480 train_time:50048ms step_avg:147.20ms
step:351/1480 train_time:50202ms step_avg:147.22ms
step:352/1480 train_time:50356ms step_avg:147.24ms
step:353/1480 train_time:50510ms step_avg:147.26ms
step:354/1480 train_time:50663ms step_avg:147.28ms
step:355/1480 train_time:50818ms step_avg:147.30ms
step:356/1480 train_time:50972ms step_avg:147.32ms
step:357/1480 train_time:51127ms step_avg:147.34ms
step:358/1480 train_time:51281ms step_avg:147.36ms
step:359/1480 train_time:51436ms step_avg:147.38ms
step:360/1480 train_time:51591ms step_avg:147.40ms
step:361/1480 train_time:51746ms step_avg:147.42ms
step:362/1480 train_time:51899ms step_avg:147.44ms
step:363/1480 train_time:52053ms step_avg:147.46ms
step:364/1480 train_time:52208ms step_avg:147.48ms
step:365/1480 train_time:52362ms step_avg:147.50ms
step:366/1480 train_time:52516ms step_avg:147.52ms
step:367/1480 train_time:52669ms step_avg:147.53ms
step:368/1480 train_time:52822ms step_avg:147.55ms
step:369/1480 train_time:52976ms step_avg:147.57ms
step:370/1480 train_time:53129ms step_avg:147.58ms
step:371/1480 train_time:53282ms step_avg:147.60ms
step:372/1480 train_time:53435ms step_avg:147.61ms
step:373/1480 train_time:53590ms step_avg:147.63ms
step:374/1480 train_time:53743ms step_avg:147.65ms
step:375/1480 train_time:53897ms step_avg:147.66ms
step:375/1480 val_loss:3.8086 train_time:53958ms step_avg:147.83ms
step:376/1480 train_time:54056ms step_avg:147.70ms
step:377/1480 train_time:54210ms step_avg:147.71ms
step:378/1480 train_time:54364ms step_avg:147.73ms
step:379/1480 train_time:54518ms step_avg:147.74ms
step:380/1480 train_time:54671ms step_avg:147.76ms
step:381/1480 train_time:54822ms step_avg:147.77ms
step:382/1480 train_time:54977ms step_avg:147.79ms
step:383/1480 train_time:55132ms step_avg:147.81ms
step:384/1480 train_time:55286ms step_avg:147.82ms
step:385/1480 train_time:55439ms step_avg:147.84ms
step:386/1480 train_time:55591ms step_avg:147.85ms
step:387/1480 train_time:55745ms step_avg:147.86ms
step:388/1480 train_time:55898ms step_avg:147.88ms
step:389/1480 train_time:56052ms step_avg:147.89ms
step:390/1480 train_time:56206ms step_avg:147.91ms
step:391/1480 train_time:56360ms step_avg:147.93ms
step:392/1480 train_time:56514ms step_avg:147.94ms
step:393/1480 train_time:56667ms step_avg:147.96ms
step:394/1480 train_time:56821ms step_avg:147.97ms
step:395/1480 train_time:56975ms step_avg:147.99ms
step:396/1480 train_time:57128ms step_avg:148.00ms
step:397/1480 train_time:57282ms step_avg:148.02ms
step:398/1480 train_time:57436ms step_avg:148.03ms
step:399/1480 train_time:57588ms step_avg:148.04ms
step:400/1480 train_time:57742ms step_avg:148.06ms
step:401/1480 train_time:57897ms step_avg:148.07ms
step:402/1480 train_time:58052ms step_avg:148.09ms
step:403/1480 train_time:58207ms step_avg:148.11ms
step:404/1480 train_time:58362ms step_avg:148.13ms
step:405/1480 train_time:58517ms step_avg:148.14ms
step:406/1480 train_time:58670ms step_avg:148.16ms
step:407/1480 train_time:58823ms step_avg:148.17ms
step:408/1480 train_time:58975ms step_avg:148.18ms
step:409/1480 train_time:59129ms step_avg:148.19ms
step:410/1480 train_time:59283ms step_avg:148.21ms
step:411/1480 train_time:59438ms step_avg:148.22ms
step:412/1480 train_time:59591ms step_avg:148.24ms
step:413/1480 train_time:59745ms step_avg:148.25ms
step:414/1480 train_time:59899ms step_avg:148.26ms
step:415/1480 train_time:60054ms step_avg:148.28ms
step:416/1480 train_time:60208ms step_avg:148.30ms
step:417/1480 train_time:60362ms step_avg:148.31ms
step:418/1480 train_time:60517ms step_avg:148.32ms
step:419/1480 train_time:60670ms step_avg:148.34ms
step:420/1480 train_time:60824ms step_avg:148.35ms
step:421/1480 train_time:60978ms step_avg:148.36ms
step:422/1480 train_time:61132ms step_avg:148.38ms
step:423/1480 train_time:61286ms step_avg:148.39ms
step:424/1480 train_time:61441ms step_avg:148.41ms
step:425/1480 train_time:61594ms step_avg:148.42ms
step:426/1480 train_time:61748ms step_avg:148.43ms
step:427/1480 train_time:61902ms step_avg:148.45ms
step:428/1480 train_time:62056ms step_avg:148.46ms
step:429/1480 train_time:62210ms step_avg:148.47ms
step:430/1480 train_time:62363ms step_avg:148.48ms
step:431/1480 train_time:62516ms step_avg:148.49ms
step:432/1480 train_time:62670ms step_avg:148.51ms
step:433/1480 train_time:62824ms step_avg:148.52ms
step:434/1480 train_time:62977ms step_avg:148.53ms
step:435/1480 train_time:63131ms step_avg:148.54ms
step:436/1480 train_time:63286ms step_avg:148.56ms
step:437/1480 train_time:63441ms step_avg:148.57ms
step:438/1480 train_time:63595ms step_avg:148.59ms
step:439/1480 train_time:63750ms step_avg:148.60ms
step:440/1480 train_time:63905ms step_avg:148.62ms
step:441/1480 train_time:64060ms step_avg:148.63ms
step:442/1480 train_time:64216ms step_avg:148.65ms
step:443/1480 train_time:64372ms step_avg:148.67ms
step:444/1480 train_time:64529ms step_avg:148.68ms
step:445/1480 train_time:64686ms step_avg:148.70ms
step:446/1480 train_time:64842ms step_avg:148.72ms
step:447/1480 train_time:64998ms step_avg:148.74ms
step:448/1480 train_time:65155ms step_avg:148.76ms
step:449/1480 train_time:65312ms step_avg:148.78ms
step:450/1480 train_time:65470ms step_avg:148.79ms
step:451/1480 train_time:65628ms step_avg:148.82ms
step:452/1480 train_time:65784ms step_avg:148.83ms
step:453/1480 train_time:65942ms step_avg:148.85ms
step:454/1480 train_time:66097ms step_avg:148.87ms
step:455/1480 train_time:66255ms step_avg:148.89ms
step:456/1480 train_time:66413ms step_avg:148.91ms
step:457/1480 train_time:66570ms step_avg:148.93ms
step:458/1480 train_time:66728ms step_avg:148.95ms
step:459/1480 train_time:66886ms step_avg:148.97ms
step:460/1480 train_time:67042ms step_avg:148.98ms
step:461/1480 train_time:67199ms step_avg:149.00ms
step:462/1480 train_time:67355ms step_avg:149.02ms
step:463/1480 train_time:67511ms step_avg:149.03ms
step:464/1480 train_time:67669ms step_avg:149.05ms
step:465/1480 train_time:67826ms step_avg:149.07ms
step:466/1480 train_time:67983ms step_avg:149.08ms
step:467/1480 train_time:68140ms step_avg:149.10ms
step:468/1480 train_time:68296ms step_avg:149.12ms
step:469/1480 train_time:68453ms step_avg:149.13ms
step:470/1480 train_time:68610ms step_avg:149.15ms
step:471/1480 train_time:68767ms step_avg:149.17ms
step:472/1480 train_time:68925ms step_avg:149.19ms
step:473/1480 train_time:69081ms step_avg:149.20ms
step:474/1480 train_time:69237ms step_avg:149.22ms
step:475/1480 train_time:69393ms step_avg:149.23ms
step:476/1480 train_time:69551ms step_avg:149.25ms
step:477/1480 train_time:69710ms step_avg:149.27ms
step:478/1480 train_time:69866ms step_avg:149.29ms
step:479/1480 train_time:70024ms step_avg:149.30ms
step:480/1480 train_time:70180ms step_avg:149.32ms
step:481/1480 train_time:70336ms step_avg:149.33ms
step:482/1480 train_time:70492ms step_avg:149.35ms
step:483/1480 train_time:70650ms step_avg:149.37ms
step:484/1480 train_time:70807ms step_avg:149.38ms
step:485/1480 train_time:70967ms step_avg:149.40ms
step:486/1480 train_time:71125ms step_avg:149.42ms
step:487/1480 train_time:71282ms step_avg:149.44ms
step:488/1480 train_time:71438ms step_avg:149.45ms
step:489/1480 train_time:71593ms step_avg:149.46ms
step:490/1480 train_time:71750ms step_avg:149.48ms
step:491/1480 train_time:71907ms step_avg:149.50ms
step:492/1480 train_time:72064ms step_avg:149.51ms
step:493/1480 train_time:72222ms step_avg:149.53ms
step:494/1480 train_time:72380ms step_avg:149.54ms
step:495/1480 train_time:72539ms step_avg:149.56ms
step:496/1480 train_time:72695ms step_avg:149.58ms
step:497/1480 train_time:72851ms step_avg:149.59ms
step:498/1480 train_time:73010ms step_avg:149.61ms
step:499/1480 train_time:73169ms step_avg:149.63ms
step:500/1480 train_time:73328ms step_avg:149.65ms
step:500/1480 val_loss:3.6860 train_time:73389ms step_avg:149.77ms
step:501/1480 train_time:73488ms step_avg:149.67ms
step:502/1480 train_time:73647ms step_avg:149.69ms
step:503/1480 train_time:73804ms step_avg:149.70ms
step:504/1480 train_time:73959ms step_avg:149.71ms
step:505/1480 train_time:74115ms step_avg:149.73ms
step:506/1480 train_time:74272ms step_avg:149.74ms
step:507/1480 train_time:74429ms step_avg:149.76ms
step:508/1480 train_time:74587ms step_avg:149.77ms
step:509/1480 train_time:74743ms step_avg:149.79ms
step:510/1480 train_time:74901ms step_avg:149.80ms
step:511/1480 train_time:75057ms step_avg:149.81ms
step:512/1480 train_time:75216ms step_avg:149.83ms
step:513/1480 train_time:75370ms step_avg:149.84ms
step:514/1480 train_time:75527ms step_avg:149.86ms
step:515/1480 train_time:75686ms step_avg:149.87ms
step:516/1480 train_time:75843ms step_avg:149.89ms
step:517/1480 train_time:76000ms step_avg:149.90ms
step:518/1480 train_time:76157ms step_avg:149.92ms
step:519/1480 train_time:76314ms step_avg:149.93ms
step:520/1480 train_time:76472ms step_avg:149.95ms
step:521/1480 train_time:76630ms step_avg:149.96ms
step:522/1480 train_time:76788ms step_avg:149.98ms
step:523/1480 train_time:76945ms step_avg:149.99ms
step:524/1480 train_time:77102ms step_avg:150.00ms
step:525/1480 train_time:77258ms step_avg:150.02ms
step:526/1480 train_time:77415ms step_avg:150.03ms
step:527/1480 train_time:77572ms step_avg:150.04ms
step:528/1480 train_time:77728ms step_avg:150.05ms
step:529/1480 train_time:77885ms step_avg:150.07ms
step:530/1480 train_time:78041ms step_avg:150.08ms
step:531/1480 train_time:78199ms step_avg:150.09ms
step:532/1480 train_time:78354ms step_avg:150.10ms
step:533/1480 train_time:78510ms step_avg:150.12ms
step:534/1480 train_time:78669ms step_avg:150.13ms
step:535/1480 train_time:78827ms step_avg:150.15ms
step:536/1480 train_time:78984ms step_avg:150.16ms
step:537/1480 train_time:79140ms step_avg:150.17ms
step:538/1480 train_time:79298ms step_avg:150.19ms
step:539/1480 train_time:79456ms step_avg:150.20ms
step:540/1480 train_time:79613ms step_avg:150.21ms
step:541/1480 train_time:79770ms step_avg:150.23ms
step:542/1480 train_time:79927ms step_avg:150.24ms
step:543/1480 train_time:80082ms step_avg:150.25ms
step:544/1480 train_time:80239ms step_avg:150.26ms
step:545/1480 train_time:80396ms step_avg:150.27ms
step:546/1480 train_time:80554ms step_avg:150.29ms
step:547/1480 train_time:80712ms step_avg:150.30ms
step:548/1480 train_time:80869ms step_avg:150.31ms
step:549/1480 train_time:81027ms step_avg:150.33ms
step:550/1480 train_time:81185ms step_avg:150.34ms
step:551/1480 train_time:81341ms step_avg:150.35ms
step:552/1480 train_time:81501ms step_avg:150.37ms
step:553/1480 train_time:81660ms step_avg:150.39ms
step:554/1480 train_time:81820ms step_avg:150.40ms
step:555/1480 train_time:81978ms step_avg:150.42ms
step:556/1480 train_time:82138ms step_avg:150.44ms
step:557/1480 train_time:82297ms step_avg:150.45ms
step:558/1480 train_time:82456ms step_avg:150.47ms
step:559/1480 train_time:82615ms step_avg:150.48ms
step:560/1480 train_time:82774ms step_avg:150.50ms
step:561/1480 train_time:82933ms step_avg:150.51ms
step:562/1480 train_time:83093ms step_avg:150.53ms
step:563/1480 train_time:83251ms step_avg:150.55ms
step:564/1480 train_time:83411ms step_avg:150.56ms
step:565/1480 train_time:83571ms step_avg:150.58ms
step:566/1480 train_time:83732ms step_avg:150.60ms
step:567/1480 train_time:83892ms step_avg:150.61ms
step:568/1480 train_time:84050ms step_avg:150.63ms
step:569/1480 train_time:84209ms step_avg:150.64ms
step:570/1480 train_time:84367ms step_avg:150.66ms
step:571/1480 train_time:84527ms step_avg:150.67ms
step:572/1480 train_time:84686ms step_avg:150.69ms
step:573/1480 train_time:84847ms step_avg:150.70ms
step:574/1480 train_time:85006ms step_avg:150.72ms
step:575/1480 train_time:85166ms step_avg:150.74ms
step:576/1480 train_time:85328ms step_avg:150.76ms
step:577/1480 train_time:85488ms step_avg:150.77ms
step:578/1480 train_time:85647ms step_avg:150.79ms
step:579/1480 train_time:85807ms step_avg:150.80ms
step:580/1480 train_time:85965ms step_avg:150.82ms
step:581/1480 train_time:86123ms step_avg:150.83ms
step:582/1480 train_time:86281ms step_avg:150.84ms
step:583/1480 train_time:86441ms step_avg:150.86ms
step:584/1480 train_time:86601ms step_avg:150.87ms
step:585/1480 train_time:86760ms step_avg:150.89ms
step:586/1480 train_time:86919ms step_avg:150.90ms
step:587/1480 train_time:87077ms step_avg:150.91ms
step:588/1480 train_time:87236ms step_avg:150.93ms
step:589/1480 train_time:87395ms step_avg:150.94ms
step:590/1480 train_time:87555ms step_avg:150.96ms
step:591/1480 train_time:87714ms step_avg:150.97ms
step:592/1480 train_time:87874ms step_avg:150.99ms
step:593/1480 train_time:88035ms step_avg:151.00ms
step:594/1480 train_time:88196ms step_avg:151.02ms
step:595/1480 train_time:88357ms step_avg:151.04ms
step:596/1480 train_time:88518ms step_avg:151.05ms
step:597/1480 train_time:88676ms step_avg:151.07ms
step:598/1480 train_time:88835ms step_avg:151.08ms
step:599/1480 train_time:88995ms step_avg:151.09ms
step:600/1480 train_time:89155ms step_avg:151.11ms
step:601/1480 train_time:89315ms step_avg:151.13ms
step:602/1480 train_time:89475ms step_avg:151.14ms
step:603/1480 train_time:89636ms step_avg:151.16ms
step:604/1480 train_time:89796ms step_avg:151.17ms
step:605/1480 train_time:89955ms step_avg:151.19ms
step:606/1480 train_time:90117ms step_avg:151.20ms
step:607/1480 train_time:90277ms step_avg:151.22ms
step:608/1480 train_time:90436ms step_avg:151.23ms
step:609/1480 train_time:90596ms step_avg:151.25ms
step:610/1480 train_time:90755ms step_avg:151.26ms
step:611/1480 train_time:90916ms step_avg:151.27ms
step:612/1480 train_time:91075ms step_avg:151.29ms
step:613/1480 train_time:91235ms step_avg:151.30ms
step:614/1480 train_time:91396ms step_avg:151.32ms
step:615/1480 train_time:91554ms step_avg:151.33ms
step:616/1480 train_time:91714ms step_avg:151.34ms
step:617/1480 train_time:91874ms step_avg:151.36ms
step:618/1480 train_time:92033ms step_avg:151.37ms
step:619/1480 train_time:92193ms step_avg:151.38ms
step:620/1480 train_time:92353ms step_avg:151.40ms
step:621/1480 train_time:92513ms step_avg:151.41ms
step:622/1480 train_time:92674ms step_avg:151.43ms
step:623/1480 train_time:92836ms step_avg:151.44ms
step:624/1480 train_time:92995ms step_avg:151.46ms
step:625/1480 train_time:93154ms step_avg:151.47ms
step:625/1480 val_loss:3.6054 train_time:93218ms step_avg:151.57ms
step:626/1480 train_time:93317ms step_avg:151.49ms
step:627/1480 train_time:93478ms step_avg:151.50ms
step:628/1480 train_time:93637ms step_avg:151.52ms
step:629/1480 train_time:93796ms step_avg:151.53ms
step:630/1480 train_time:93954ms step_avg:151.54ms
step:631/1480 train_time:94113ms step_avg:151.55ms
step:632/1480 train_time:94272ms step_avg:151.56ms
step:633/1480 train_time:94433ms step_avg:151.58ms
step:634/1480 train_time:94591ms step_avg:151.59ms
step:635/1480 train_time:94748ms step_avg:151.60ms
step:636/1480 train_time:94907ms step_avg:151.61ms
step:637/1480 train_time:95067ms step_avg:151.62ms
step:638/1480 train_time:95225ms step_avg:151.63ms
step:639/1480 train_time:95384ms step_avg:151.64ms
step:640/1480 train_time:95545ms step_avg:151.66ms
step:641/1480 train_time:95705ms step_avg:151.67ms
step:642/1480 train_time:95864ms step_avg:151.68ms
step:643/1480 train_time:96024ms step_avg:151.70ms
step:644/1480 train_time:96183ms step_avg:151.71ms
step:645/1480 train_time:96342ms step_avg:151.72ms
step:646/1480 train_time:96502ms step_avg:151.73ms
step:647/1480 train_time:96662ms step_avg:151.75ms
step:648/1480 train_time:96823ms step_avg:151.76ms
step:649/1480 train_time:96983ms step_avg:151.77ms
step:650/1480 train_time:97142ms step_avg:151.78ms
step:651/1480 train_time:97303ms step_avg:151.80ms
step:652/1480 train_time:97463ms step_avg:151.81ms
step:653/1480 train_time:97623ms step_avg:151.82ms
step:654/1480 train_time:97783ms step_avg:151.84ms
step:655/1480 train_time:97943ms step_avg:151.85ms
step:656/1480 train_time:98103ms step_avg:151.86ms
step:657/1480 train_time:98263ms step_avg:151.88ms
step:658/1480 train_time:98423ms step_avg:151.89ms
step:659/1480 train_time:98586ms step_avg:151.90ms
step:660/1480 train_time:98748ms step_avg:151.92ms
step:661/1480 train_time:98911ms step_avg:151.94ms
step:662/1480 train_time:99071ms step_avg:151.95ms
step:663/1480 train_time:99230ms step_avg:151.96ms
step:664/1480 train_time:99391ms step_avg:151.97ms
step:665/1480 train_time:99554ms step_avg:151.99ms
step:666/1480 train_time:99714ms step_avg:152.00ms
step:667/1480 train_time:99877ms step_avg:152.02ms
step:668/1480 train_time:100039ms step_avg:152.03ms
step:669/1480 train_time:100201ms step_avg:152.05ms
step:670/1480 train_time:100362ms step_avg:152.06ms
step:671/1480 train_time:100524ms step_avg:152.08ms
step:672/1480 train_time:100685ms step_avg:152.09ms
step:673/1480 train_time:100847ms step_avg:152.11ms
step:674/1480 train_time:101009ms step_avg:152.12ms
step:675/1480 train_time:101171ms step_avg:152.14ms
step:676/1480 train_time:101331ms step_avg:152.15ms
step:677/1480 train_time:101493ms step_avg:152.16ms
step:678/1480 train_time:101652ms step_avg:152.17ms
step:679/1480 train_time:101815ms step_avg:152.19ms
step:680/1480 train_time:101978ms step_avg:152.21ms
step:681/1480 train_time:102140ms step_avg:152.22ms
step:682/1480 train_time:102304ms step_avg:152.24ms
step:683/1480 train_time:102466ms step_avg:152.25ms
step:684/1480 train_time:102627ms step_avg:152.26ms
step:685/1480 train_time:102789ms step_avg:152.28ms
step:686/1480 train_time:102949ms step_avg:152.29ms
step:687/1480 train_time:103109ms step_avg:152.30ms
step:688/1480 train_time:103272ms step_avg:152.32ms
step:689/1480 train_time:103436ms step_avg:152.34ms
step:690/1480 train_time:103599ms step_avg:152.35ms
step:691/1480 train_time:103761ms step_avg:152.37ms
step:692/1480 train_time:103924ms step_avg:152.38ms
step:693/1480 train_time:104085ms step_avg:152.39ms
step:694/1480 train_time:104246ms step_avg:152.41ms
step:695/1480 train_time:104406ms step_avg:152.42ms
step:696/1480 train_time:104567ms step_avg:152.43ms
step:697/1480 train_time:104730ms step_avg:152.45ms
step:698/1480 train_time:104890ms step_avg:152.46ms
step:699/1480 train_time:105051ms step_avg:152.47ms
step:700/1480 train_time:105213ms step_avg:152.48ms
step:701/1480 train_time:105374ms step_avg:152.49ms
step:702/1480 train_time:105534ms step_avg:152.51ms
step:703/1480 train_time:105693ms step_avg:152.52ms
step:704/1480 train_time:105852ms step_avg:152.52ms
step:705/1480 train_time:106016ms step_avg:152.54ms
step:706/1480 train_time:106179ms step_avg:152.56ms
step:707/1480 train_time:106342ms step_avg:152.57ms
step:708/1480 train_time:106504ms step_avg:152.58ms
step:709/1480 train_time:106666ms step_avg:152.60ms
step:710/1480 train_time:106826ms step_avg:152.61ms
step:711/1480 train_time:106989ms step_avg:152.62ms
step:712/1480 train_time:107151ms step_avg:152.64ms
step:713/1480 train_time:107315ms step_avg:152.65ms
step:714/1480 train_time:107475ms step_avg:152.66ms
step:715/1480 train_time:107635ms step_avg:152.67ms
step:716/1480 train_time:107796ms step_avg:152.69ms
step:717/1480 train_time:107960ms step_avg:152.70ms
step:718/1480 train_time:108121ms step_avg:152.71ms
step:719/1480 train_time:108281ms step_avg:152.72ms
step:720/1480 train_time:108445ms step_avg:152.74ms
step:721/1480 train_time:108608ms step_avg:152.75ms
step:722/1480 train_time:108769ms step_avg:152.76ms
step:723/1480 train_time:108929ms step_avg:152.78ms
step:724/1480 train_time:109091ms step_avg:152.79ms
step:725/1480 train_time:109254ms step_avg:152.80ms
step:726/1480 train_time:109420ms step_avg:152.82ms
step:727/1480 train_time:109584ms step_avg:152.84ms
step:728/1480 train_time:109745ms step_avg:152.85ms
step:729/1480 train_time:109906ms step_avg:152.86ms
step:730/1480 train_time:110069ms step_avg:152.87ms
step:731/1480 train_time:110230ms step_avg:152.88ms
step:732/1480 train_time:110389ms step_avg:152.89ms
step:733/1480 train_time:110550ms step_avg:152.91ms
step:734/1480 train_time:110712ms step_avg:152.92ms
step:735/1480 train_time:110872ms step_avg:152.93ms
step:736/1480 train_time:111033ms step_avg:152.94ms
step:737/1480 train_time:111193ms step_avg:152.95ms
step:738/1480 train_time:111354ms step_avg:152.96ms
step:739/1480 train_time:111515ms step_avg:152.97ms
step:740/1480 train_time:111682ms step_avg:152.99ms
step:741/1480 train_time:111844ms step_avg:153.00ms
step:742/1480 train_time:112006ms step_avg:153.01ms
step:743/1480 train_time:112167ms step_avg:153.02ms
step:744/1480 train_time:112330ms step_avg:153.04ms
step:745/1480 train_time:112493ms step_avg:153.05ms
step:746/1480 train_time:112653ms step_avg:153.06ms
step:747/1480 train_time:112814ms step_avg:153.07ms
step:748/1480 train_time:112981ms step_avg:153.09ms
step:749/1480 train_time:113146ms step_avg:153.11ms
step:750/1480 train_time:113306ms step_avg:153.12ms
step:750/1480 val_loss:3.5500 train_time:113369ms step_avg:153.20ms
step:751/1480 train_time:113471ms step_avg:153.13ms
step:752/1480 train_time:113633ms step_avg:153.14ms
step:753/1480 train_time:113794ms step_avg:153.15ms
step:754/1480 train_time:113955ms step_avg:153.17ms
step:755/1480 train_time:114116ms step_avg:153.18ms
step:756/1480 train_time:114278ms step_avg:153.19ms
step:757/1480 train_time:114443ms step_avg:153.20ms
step:758/1480 train_time:114605ms step_avg:153.21ms
step:759/1480 train_time:114767ms step_avg:153.23ms
step:760/1480 train_time:114928ms step_avg:153.24ms
step:761/1480 train_time:115089ms step_avg:153.25ms
step:762/1480 train_time:115250ms step_avg:153.26ms
step:763/1480 train_time:115411ms step_avg:153.27ms
step:764/1480 train_time:115572ms step_avg:153.28ms
step:765/1480 train_time:115733ms step_avg:153.29ms
step:766/1480 train_time:115895ms step_avg:153.30ms
step:767/1480 train_time:116058ms step_avg:153.31ms
step:768/1480 train_time:116222ms step_avg:153.33ms
step:769/1480 train_time:116385ms step_avg:153.34ms
step:770/1480 train_time:116548ms step_avg:153.35ms
step:771/1480 train_time:116711ms step_avg:153.37ms
step:772/1480 train_time:116873ms step_avg:153.38ms
step:773/1480 train_time:117036ms step_avg:153.39ms
step:774/1480 train_time:117199ms step_avg:153.40ms
step:775/1480 train_time:117364ms step_avg:153.42ms
step:776/1480 train_time:117528ms step_avg:153.43ms
step:777/1480 train_time:117692ms step_avg:153.44ms
step:778/1480 train_time:117856ms step_avg:153.46ms
step:779/1480 train_time:118018ms step_avg:153.47ms
step:780/1480 train_time:118183ms step_avg:153.48ms
step:781/1480 train_time:118347ms step_avg:153.50ms
step:782/1480 train_time:118509ms step_avg:153.51ms
step:783/1480 train_time:118670ms step_avg:153.52ms
step:784/1480 train_time:118834ms step_avg:153.53ms
step:785/1480 train_time:118996ms step_avg:153.54ms
step:786/1480 train_time:119163ms step_avg:153.56ms
step:787/1480 train_time:119326ms step_avg:153.57ms
step:788/1480 train_time:119490ms step_avg:153.59ms
step:789/1480 train_time:119651ms step_avg:153.60ms
step:790/1480 train_time:119816ms step_avg:153.61ms
step:791/1480 train_time:119983ms step_avg:153.63ms
step:792/1480 train_time:120148ms step_avg:153.64ms
step:793/1480 train_time:120309ms step_avg:153.65ms
step:794/1480 train_time:120472ms step_avg:153.66ms
step:795/1480 train_time:120643ms step_avg:153.69ms
step:796/1480 train_time:120811ms step_avg:153.70ms
step:797/1480 train_time:120974ms step_avg:153.72ms
step:798/1480 train_time:121139ms step_avg:153.73ms
step:799/1480 train_time:121305ms step_avg:153.75ms
step:800/1480 train_time:121468ms step_avg:153.76ms
step:801/1480 train_time:121631ms step_avg:153.77ms
step:802/1480 train_time:121798ms step_avg:153.79ms
step:803/1480 train_time:121962ms step_avg:153.80ms
step:804/1480 train_time:122125ms step_avg:153.81ms
step:805/1480 train_time:122289ms step_avg:153.82ms
step:806/1480 train_time:122450ms step_avg:153.83ms
step:807/1480 train_time:122611ms step_avg:153.84ms
step:808/1480 train_time:122775ms step_avg:153.85ms
step:809/1480 train_time:122939ms step_avg:153.87ms
step:810/1480 train_time:123102ms step_avg:153.88ms
step:811/1480 train_time:123265ms step_avg:153.89ms
step:812/1480 train_time:123428ms step_avg:153.90ms
step:813/1480 train_time:123589ms step_avg:153.91ms
step:814/1480 train_time:123752ms step_avg:153.92ms
step:815/1480 train_time:123914ms step_avg:153.93ms
step:816/1480 train_time:124080ms step_avg:153.95ms
step:817/1480 train_time:124244ms step_avg:153.96ms
step:818/1480 train_time:124406ms step_avg:153.97ms
step:819/1480 train_time:124570ms step_avg:153.98ms
step:820/1480 train_time:124733ms step_avg:153.99ms
step:821/1480 train_time:124892ms step_avg:154.00ms
step:822/1480 train_time:125058ms step_avg:154.01ms
step:823/1480 train_time:125223ms step_avg:154.03ms
step:824/1480 train_time:125386ms step_avg:154.04ms
step:825/1480 train_time:125550ms step_avg:154.05ms
step:826/1480 train_time:125716ms step_avg:154.06ms
step:827/1480 train_time:125880ms step_avg:154.08ms
step:828/1480 train_time:126044ms step_avg:154.09ms
step:829/1480 train_time:126207ms step_avg:154.10ms
step:830/1480 train_time:126371ms step_avg:154.11ms
step:831/1480 train_time:126535ms step_avg:154.12ms
step:832/1480 train_time:126699ms step_avg:154.14ms
step:833/1480 train_time:126865ms step_avg:154.15ms
step:834/1480 train_time:127028ms step_avg:154.16ms
step:835/1480 train_time:127190ms step_avg:154.17ms
step:836/1480 train_time:127356ms step_avg:154.18ms
step:837/1480 train_time:127519ms step_avg:154.19ms
step:838/1480 train_time:127683ms step_avg:154.21ms
step:839/1480 train_time:127847ms step_avg:154.22ms
step:840/1480 train_time:128007ms step_avg:154.23ms
step:841/1480 train_time:128168ms step_avg:154.23ms
step:842/1480 train_time:128331ms step_avg:154.24ms
step:843/1480 train_time:128493ms step_avg:154.25ms
step:844/1480 train_time:128654ms step_avg:154.26ms
step:845/1480 train_time:128819ms step_avg:154.27ms
step:846/1480 train_time:128984ms step_avg:154.29ms
step:847/1480 train_time:129148ms step_avg:154.30ms
step:848/1480 train_time:129309ms step_avg:154.31ms
step:849/1480 train_time:129472ms step_avg:154.32ms
step:850/1480 train_time:129636ms step_avg:154.33ms
step:851/1480 train_time:129802ms step_avg:154.34ms
step:852/1480 train_time:129965ms step_avg:154.35ms
step:853/1480 train_time:130127ms step_avg:154.36ms
step:854/1480 train_time:130292ms step_avg:154.37ms
step:855/1480 train_time:130455ms step_avg:154.39ms
step:856/1480 train_time:130618ms step_avg:154.39ms
step:857/1480 train_time:130783ms step_avg:154.41ms
step:858/1480 train_time:130948ms step_avg:154.42ms
step:859/1480 train_time:131110ms step_avg:154.43ms
step:860/1480 train_time:131271ms step_avg:154.44ms
step:861/1480 train_time:131438ms step_avg:154.45ms
step:862/1480 train_time:131608ms step_avg:154.47ms
step:863/1480 train_time:131775ms step_avg:154.48ms
step:864/1480 train_time:131939ms step_avg:154.50ms
step:865/1480 train_time:132101ms step_avg:154.50ms
step:866/1480 train_time:132269ms step_avg:154.52ms
step:867/1480 train_time:132432ms step_avg:154.53ms
step:868/1480 train_time:132594ms step_avg:154.54ms
step:869/1480 train_time:132757ms step_avg:154.55ms
step:870/1480 train_time:132921ms step_avg:154.56ms
step:871/1480 train_time:133084ms step_avg:154.57ms
step:872/1480 train_time:133248ms step_avg:154.58ms
step:873/1480 train_time:133411ms step_avg:154.59ms
step:874/1480 train_time:133577ms step_avg:154.60ms
step:875/1480 train_time:133743ms step_avg:154.62ms
step:875/1480 val_loss:3.5044 train_time:133808ms step_avg:154.69ms
step:876/1480 train_time:133910ms step_avg:154.63ms
step:877/1480 train_time:134075ms step_avg:154.64ms
step:878/1480 train_time:134238ms step_avg:154.65ms
step:879/1480 train_time:134402ms step_avg:154.66ms
step:880/1480 train_time:134565ms step_avg:154.67ms
step:881/1480 train_time:134727ms step_avg:154.68ms
step:882/1480 train_time:134893ms step_avg:154.69ms
step:883/1480 train_time:135059ms step_avg:154.71ms
step:884/1480 train_time:135224ms step_avg:154.72ms
step:885/1480 train_time:135390ms step_avg:154.73ms
step:886/1480 train_time:135556ms step_avg:154.74ms
step:887/1480 train_time:135723ms step_avg:154.76ms
step:888/1480 train_time:135897ms step_avg:154.78ms
step:889/1480 train_time:136066ms step_avg:154.80ms
step:890/1480 train_time:136228ms step_avg:154.80ms
step:891/1480 train_time:136396ms step_avg:154.82ms
step:892/1480 train_time:136561ms step_avg:154.83ms
step:893/1480 train_time:136722ms step_avg:154.84ms
step:894/1480 train_time:136890ms step_avg:154.85ms
step:895/1480 train_time:137055ms step_avg:154.86ms
step:896/1480 train_time:137221ms step_avg:154.88ms
step:897/1480 train_time:137388ms step_avg:154.89ms
step:898/1480 train_time:137556ms step_avg:154.91ms
step:899/1480 train_time:137719ms step_avg:154.91ms
step:900/1480 train_time:137882ms step_avg:154.92ms
step:901/1480 train_time:138046ms step_avg:154.93ms
step:902/1480 train_time:138212ms step_avg:154.95ms
step:903/1480 train_time:138383ms step_avg:154.96ms
step:904/1480 train_time:138548ms step_avg:154.98ms
step:905/1480 train_time:138712ms step_avg:154.99ms
step:906/1480 train_time:138879ms step_avg:155.00ms
step:907/1480 train_time:139047ms step_avg:155.01ms
step:908/1480 train_time:139211ms step_avg:155.02ms
step:909/1480 train_time:139376ms step_avg:155.03ms
step:910/1480 train_time:139546ms step_avg:155.05ms
step:911/1480 train_time:139711ms step_avg:155.06ms
step:912/1480 train_time:139877ms step_avg:155.07ms
step:913/1480 train_time:140043ms step_avg:155.09ms
step:914/1480 train_time:140210ms step_avg:155.10ms
step:915/1480 train_time:140378ms step_avg:155.11ms
step:916/1480 train_time:140544ms step_avg:155.13ms
step:917/1480 train_time:140706ms step_avg:155.13ms
step:918/1480 train_time:140877ms step_avg:155.15ms
step:919/1480 train_time:141047ms step_avg:155.17ms
step:920/1480 train_time:141213ms step_avg:155.18ms
step:921/1480 train_time:141379ms step_avg:155.19ms
step:922/1480 train_time:141545ms step_avg:155.20ms
step:923/1480 train_time:141709ms step_avg:155.21ms
step:924/1480 train_time:141875ms step_avg:155.22ms
step:925/1480 train_time:142039ms step_avg:155.23ms
step:926/1480 train_time:142202ms step_avg:155.24ms
step:927/1480 train_time:142368ms step_avg:155.25ms
step:928/1480 train_time:142535ms step_avg:155.27ms
step:929/1480 train_time:142700ms step_avg:155.28ms
step:930/1480 train_time:142864ms step_avg:155.29ms
step:931/1480 train_time:143027ms step_avg:155.30ms
step:932/1480 train_time:143194ms step_avg:155.31ms
step:933/1480 train_time:143361ms step_avg:155.32ms
step:934/1480 train_time:143530ms step_avg:155.34ms
step:935/1480 train_time:143703ms step_avg:155.35ms
step:936/1480 train_time:143872ms step_avg:155.37ms
step:937/1480 train_time:144040ms step_avg:155.38ms
step:938/1480 train_time:144201ms step_avg:155.39ms
step:939/1480 train_time:144372ms step_avg:155.41ms
step:940/1480 train_time:144539ms step_avg:155.42ms
step:941/1480 train_time:144702ms step_avg:155.43ms
step:942/1480 train_time:144869ms step_avg:155.44ms
step:943/1480 train_time:145039ms step_avg:155.45ms
step:944/1480 train_time:145211ms step_avg:155.47ms
step:945/1480 train_time:145376ms step_avg:155.48ms
step:946/1480 train_time:145543ms step_avg:155.49ms
step:947/1480 train_time:145710ms step_avg:155.51ms
step:948/1480 train_time:145876ms step_avg:155.52ms
step:949/1480 train_time:146041ms step_avg:155.53ms
step:950/1480 train_time:146204ms step_avg:155.54ms
step:951/1480 train_time:146373ms step_avg:155.55ms
step:952/1480 train_time:146539ms step_avg:155.56ms
step:953/1480 train_time:146707ms step_avg:155.57ms
step:954/1480 train_time:146875ms step_avg:155.59ms
step:955/1480 train_time:147038ms step_avg:155.60ms
step:956/1480 train_time:147202ms step_avg:155.60ms
step:957/1480 train_time:147371ms step_avg:155.62ms
step:958/1480 train_time:147539ms step_avg:155.63ms
step:959/1480 train_time:147703ms step_avg:155.64ms
step:960/1480 train_time:147871ms step_avg:155.65ms
step:961/1480 train_time:148036ms step_avg:155.66ms
step:962/1480 train_time:148200ms step_avg:155.67ms
step:963/1480 train_time:148366ms step_avg:155.68ms
step:964/1480 train_time:148535ms step_avg:155.70ms
step:965/1480 train_time:148698ms step_avg:155.70ms
step:966/1480 train_time:148863ms step_avg:155.71ms
step:967/1480 train_time:149028ms step_avg:155.72ms
step:968/1480 train_time:149193ms step_avg:155.73ms
step:969/1480 train_time:149358ms step_avg:155.74ms
step:970/1480 train_time:149521ms step_avg:155.75ms
step:971/1480 train_time:149688ms step_avg:155.76ms
step:972/1480 train_time:149852ms step_avg:155.77ms
step:973/1480 train_time:150016ms step_avg:155.78ms
step:974/1480 train_time:150185ms step_avg:155.79ms
step:975/1480 train_time:150351ms step_avg:155.80ms
step:976/1480 train_time:150517ms step_avg:155.81ms
step:977/1480 train_time:150680ms step_avg:155.82ms
step:978/1480 train_time:150846ms step_avg:155.83ms
step:979/1480 train_time:151015ms step_avg:155.85ms
step:980/1480 train_time:151181ms step_avg:155.86ms
step:981/1480 train_time:151352ms step_avg:155.87ms
step:982/1480 train_time:151515ms step_avg:155.88ms
step:983/1480 train_time:151680ms step_avg:155.89ms
step:984/1480 train_time:151843ms step_avg:155.90ms
step:985/1480 train_time:152014ms step_avg:155.91ms
step:986/1480 train_time:152180ms step_avg:155.92ms
step:987/1480 train_time:152344ms step_avg:155.93ms
step:988/1480 train_time:152513ms step_avg:155.94ms
step:989/1480 train_time:152678ms step_avg:155.95ms
step:990/1480 train_time:152846ms step_avg:155.97ms
step:991/1480 train_time:153015ms step_avg:155.98ms
step:992/1480 train_time:153189ms step_avg:156.00ms
step:993/1480 train_time:153364ms step_avg:156.02ms
step:994/1480 train_time:153528ms step_avg:156.02ms
step:995/1480 train_time:153693ms step_avg:156.03ms
step:996/1480 train_time:153856ms step_avg:156.04ms
step:997/1480 train_time:154021ms step_avg:156.05ms
step:998/1480 train_time:154185ms step_avg:156.06ms
step:999/1480 train_time:154351ms step_avg:156.07ms
step:1000/1480 train_time:154519ms step_avg:156.08ms
step:1000/1480 val_loss:3.4418 train_time:154587ms step_avg:156.15ms
step:1001/1480 train_time:154688ms step_avg:156.09ms
step:1002/1480 train_time:154855ms step_avg:156.10ms
step:1003/1480 train_time:155028ms step_avg:156.12ms
step:1004/1480 train_time:155197ms step_avg:156.13ms
step:1005/1480 train_time:155365ms step_avg:156.15ms
step:1006/1480 train_time:155531ms step_avg:156.16ms
step:1007/1480 train_time:155697ms step_avg:156.17ms
step:1008/1480 train_time:155863ms step_avg:156.18ms
step:1009/1480 train_time:156036ms step_avg:156.19ms
step:1010/1480 train_time:156201ms step_avg:156.20ms
step:1011/1480 train_time:156366ms step_avg:156.21ms
step:1012/1480 train_time:156531ms step_avg:156.22ms
step:1013/1480 train_time:156702ms step_avg:156.23ms
step:1014/1480 train_time:156869ms step_avg:156.24ms
step:1015/1480 train_time:157039ms step_avg:156.26ms
step:1016/1480 train_time:157209ms step_avg:156.27ms
step:1017/1480 train_time:157380ms step_avg:156.29ms
step:1018/1480 train_time:157548ms step_avg:156.30ms
step:1019/1480 train_time:157717ms step_avg:156.31ms
step:1020/1480 train_time:157887ms step_avg:156.32ms
step:1021/1480 train_time:158051ms step_avg:156.33ms
step:1022/1480 train_time:158218ms step_avg:156.34ms
step:1023/1480 train_time:158385ms step_avg:156.35ms
step:1024/1480 train_time:158552ms step_avg:156.36ms
step:1025/1480 train_time:158723ms step_avg:156.38ms
step:1026/1480 train_time:158889ms step_avg:156.39ms
step:1027/1480 train_time:159054ms step_avg:156.39ms
step:1028/1480 train_time:159229ms step_avg:156.41ms
step:1029/1480 train_time:159403ms step_avg:156.43ms
step:1030/1480 train_time:159571ms step_avg:156.44ms
step:1031/1480 train_time:159735ms step_avg:156.45ms
step:1032/1480 train_time:159908ms step_avg:156.47ms
step:1033/1480 train_time:160073ms step_avg:156.47ms
step:1034/1480 train_time:160242ms step_avg:156.49ms
step:1035/1480 train_time:160409ms step_avg:156.50ms
step:1036/1480 train_time:160574ms step_avg:156.50ms
step:1037/1480 train_time:160745ms step_avg:156.52ms
step:1038/1480 train_time:160914ms step_avg:156.53ms
step:1039/1480 train_time:161085ms step_avg:156.55ms
step:1040/1480 train_time:161251ms step_avg:156.55ms
step:1041/1480 train_time:161418ms step_avg:156.56ms
step:1042/1480 train_time:161584ms step_avg:156.57ms
step:1043/1480 train_time:161750ms step_avg:156.58ms
step:1044/1480 train_time:161916ms step_avg:156.59ms
step:1045/1480 train_time:162087ms step_avg:156.61ms
step:1046/1480 train_time:162254ms step_avg:156.62ms
step:1047/1480 train_time:162421ms step_avg:156.63ms
step:1048/1480 train_time:162586ms step_avg:156.63ms
step:1049/1480 train_time:162752ms step_avg:156.64ms
step:1050/1480 train_time:162923ms step_avg:156.66ms
step:1051/1480 train_time:163092ms step_avg:156.67ms
step:1052/1480 train_time:163261ms step_avg:156.68ms
step:1053/1480 train_time:163427ms step_avg:156.69ms
step:1054/1480 train_time:163594ms step_avg:156.70ms
step:1055/1480 train_time:163760ms step_avg:156.71ms
step:1056/1480 train_time:163925ms step_avg:156.72ms
step:1057/1480 train_time:164090ms step_avg:156.72ms
step:1058/1480 train_time:164259ms step_avg:156.74ms
step:1059/1480 train_time:164433ms step_avg:156.75ms
step:1060/1480 train_time:164601ms step_avg:156.76ms
step:1061/1480 train_time:164766ms step_avg:156.77ms
step:1062/1480 train_time:164931ms step_avg:156.78ms
step:1063/1480 train_time:165095ms step_avg:156.79ms
step:1064/1480 train_time:165259ms step_avg:156.79ms
step:1065/1480 train_time:165428ms step_avg:156.80ms
step:1066/1480 train_time:165595ms step_avg:156.81ms
step:1067/1480 train_time:165766ms step_avg:156.83ms
step:1068/1480 train_time:165933ms step_avg:156.84ms
step:1069/1480 train_time:166105ms step_avg:156.85ms
step:1070/1480 train_time:166270ms step_avg:156.86ms
step:1071/1480 train_time:166445ms step_avg:156.88ms
step:1072/1480 train_time:166610ms step_avg:156.88ms
step:1073/1480 train_time:166773ms step_avg:156.89ms
step:1074/1480 train_time:166940ms step_avg:156.90ms
step:1075/1480 train_time:167111ms step_avg:156.91ms
step:1076/1480 train_time:167277ms step_avg:156.92ms
step:1077/1480 train_time:167444ms step_avg:156.93ms
step:1078/1480 train_time:167618ms step_avg:156.95ms
step:1079/1480 train_time:167790ms step_avg:156.96ms
step:1080/1480 train_time:167961ms step_avg:156.97ms
step:1081/1480 train_time:168128ms step_avg:156.98ms
step:1082/1480 train_time:168294ms step_avg:156.99ms
step:1083/1480 train_time:168460ms step_avg:157.00ms
step:1084/1480 train_time:168628ms step_avg:157.01ms
step:1085/1480 train_time:168797ms step_avg:157.02ms
step:1086/1480 train_time:168967ms step_avg:157.03ms
step:1087/1480 train_time:169132ms step_avg:157.04ms
step:1088/1480 train_time:169300ms step_avg:157.05ms
step:1089/1480 train_time:169471ms step_avg:157.06ms
step:1090/1480 train_time:169645ms step_avg:157.08ms
step:1091/1480 train_time:169813ms step_avg:157.09ms
step:1092/1480 train_time:169982ms step_avg:157.10ms
step:1093/1480 train_time:170151ms step_avg:157.11ms
step:1094/1480 train_time:170316ms step_avg:157.12ms
step:1095/1480 train_time:170482ms step_avg:157.13ms
step:1096/1480 train_time:170651ms step_avg:157.14ms
step:1097/1480 train_time:170821ms step_avg:157.15ms
step:1098/1480 train_time:170991ms step_avg:157.16ms
step:1099/1480 train_time:171163ms step_avg:157.17ms
step:1100/1480 train_time:171334ms step_avg:157.19ms
step:1101/1480 train_time:171506ms step_avg:157.20ms
step:1102/1480 train_time:171676ms step_avg:157.21ms
step:1103/1480 train_time:171855ms step_avg:157.23ms
step:1104/1480 train_time:172023ms step_avg:157.24ms
step:1105/1480 train_time:172192ms step_avg:157.25ms
step:1106/1480 train_time:172360ms step_avg:157.26ms
step:1107/1480 train_time:172529ms step_avg:157.27ms
step:1108/1480 train_time:172693ms step_avg:157.28ms
step:1109/1480 train_time:172860ms step_avg:157.29ms
step:1110/1480 train_time:173027ms step_avg:157.30ms
step:1111/1480 train_time:173194ms step_avg:157.31ms
step:1112/1480 train_time:173366ms step_avg:157.32ms
step:1113/1480 train_time:173547ms step_avg:157.34ms
step:1114/1480 train_time:173718ms step_avg:157.35ms
step:1115/1480 train_time:173890ms step_avg:157.37ms
step:1116/1480 train_time:174057ms step_avg:157.38ms
step:1117/1480 train_time:174231ms step_avg:157.39ms
step:1118/1480 train_time:174405ms step_avg:157.41ms
step:1119/1480 train_time:174570ms step_avg:157.41ms
step:1120/1480 train_time:174740ms step_avg:157.42ms
step:1121/1480 train_time:174910ms step_avg:157.43ms
step:1122/1480 train_time:175075ms step_avg:157.44ms
step:1123/1480 train_time:175243ms step_avg:157.45ms
step:1124/1480 train_time:175411ms step_avg:157.46ms
step:1125/1480 train_time:175579ms step_avg:157.47ms
step:1125/1480 val_loss:3.3852 train_time:175646ms step_avg:157.53ms
step:1126/1480 train_time:175749ms step_avg:157.48ms
step:1127/1480 train_time:175917ms step_avg:157.49ms
step:1128/1480 train_time:176090ms step_avg:157.50ms
step:1129/1480 train_time:176264ms step_avg:157.52ms
step:1130/1480 train_time:176434ms step_avg:157.53ms
step:1131/1480 train_time:176612ms step_avg:157.55ms
step:1132/1480 train_time:176776ms step_avg:157.55ms
step:1133/1480 train_time:176949ms step_avg:157.57ms
step:1134/1480 train_time:177120ms step_avg:157.58ms
step:1135/1480 train_time:177290ms step_avg:157.59ms
step:1136/1480 train_time:177459ms step_avg:157.60ms
step:1137/1480 train_time:177629ms step_avg:157.61ms
step:1138/1480 train_time:177799ms step_avg:157.62ms
step:1139/1480 train_time:177968ms step_avg:157.63ms
step:1140/1480 train_time:178136ms step_avg:157.64ms
step:1141/1480 train_time:178310ms step_avg:157.66ms
step:1142/1480 train_time:178477ms step_avg:157.67ms
step:1143/1480 train_time:178649ms step_avg:157.68ms
step:1144/1480 train_time:178816ms step_avg:157.69ms
step:1145/1480 train_time:178982ms step_avg:157.69ms
step:1146/1480 train_time:179154ms step_avg:157.71ms
step:1147/1480 train_time:179322ms step_avg:157.71ms
step:1148/1480 train_time:179490ms step_avg:157.72ms
step:1149/1480 train_time:179659ms step_avg:157.73ms
step:1150/1480 train_time:179828ms step_avg:157.74ms
step:1151/1480 train_time:179999ms step_avg:157.76ms
step:1152/1480 train_time:180172ms step_avg:157.77ms
step:1153/1480 train_time:180345ms step_avg:157.78ms
step:1154/1480 train_time:180512ms step_avg:157.79ms
step:1155/1480 train_time:180685ms step_avg:157.80ms
step:1156/1480 train_time:180866ms step_avg:157.82ms
step:1157/1480 train_time:181034ms step_avg:157.83ms
step:1158/1480 train_time:181201ms step_avg:157.84ms
step:1159/1480 train_time:181370ms step_avg:157.85ms
step:1160/1480 train_time:181536ms step_avg:157.86ms
step:1161/1480 train_time:181706ms step_avg:157.87ms
step:1162/1480 train_time:181876ms step_avg:157.88ms
step:1163/1480 train_time:182047ms step_avg:157.89ms
step:1164/1480 train_time:182215ms step_avg:157.90ms
step:1165/1480 train_time:182380ms step_avg:157.90ms
step:1166/1480 train_time:182550ms step_avg:157.92ms
step:1167/1480 train_time:182717ms step_avg:157.92ms
step:1168/1480 train_time:182886ms step_avg:157.93ms
step:1169/1480 train_time:183055ms step_avg:157.94ms
step:1170/1480 train_time:183224ms step_avg:157.95ms
step:1171/1480 train_time:183391ms step_avg:157.96ms
step:1172/1480 train_time:183558ms step_avg:157.97ms
step:1173/1480 train_time:183729ms step_avg:157.98ms
step:1174/1480 train_time:183910ms step_avg:158.00ms
step:1175/1480 train_time:184081ms step_avg:158.01ms
step:1176/1480 train_time:184254ms step_avg:158.02ms
step:1177/1480 train_time:184431ms step_avg:158.04ms
step:1178/1480 train_time:184599ms step_avg:158.05ms
step:1179/1480 train_time:184765ms step_avg:158.05ms
step:1180/1480 train_time:184945ms step_avg:158.07ms
step:1181/1480 train_time:185115ms step_avg:158.08ms
step:1182/1480 train_time:185284ms step_avg:158.09ms
step:1183/1480 train_time:185456ms step_avg:158.10ms
step:1184/1480 train_time:185623ms step_avg:158.11ms
step:1185/1480 train_time:185796ms step_avg:158.12ms
step:1186/1480 train_time:185967ms step_avg:158.13ms
step:1187/1480 train_time:186149ms step_avg:158.16ms
step:1188/1480 train_time:186315ms step_avg:158.16ms
step:1189/1480 train_time:186486ms step_avg:158.17ms
step:1190/1480 train_time:186654ms step_avg:158.18ms
step:1191/1480 train_time:186826ms step_avg:158.19ms
step:1192/1480 train_time:186992ms step_avg:158.20ms
step:1193/1480 train_time:187159ms step_avg:158.21ms
step:1194/1480 train_time:187329ms step_avg:158.22ms
step:1195/1480 train_time:187503ms step_avg:158.23ms
step:1196/1480 train_time:187686ms step_avg:158.25ms
step:1197/1480 train_time:187857ms step_avg:158.26ms
step:1198/1480 train_time:188038ms step_avg:158.28ms
step:1199/1480 train_time:188210ms step_avg:158.29ms
step:1200/1480 train_time:188377ms step_avg:158.30ms
step:1201/1480 train_time:188545ms step_avg:158.31ms
step:1202/1480 train_time:188726ms step_avg:158.33ms
step:1203/1480 train_time:188902ms step_avg:158.34ms
step:1204/1480 train_time:189077ms step_avg:158.36ms
step:1205/1480 train_time:189244ms step_avg:158.36ms
step:1206/1480 train_time:189411ms step_avg:158.37ms
step:1207/1480 train_time:189579ms step_avg:158.38ms
step:1208/1480 train_time:189748ms step_avg:158.39ms
step:1209/1480 train_time:189920ms step_avg:158.40ms
step:1210/1480 train_time:190095ms step_avg:158.41ms
step:1211/1480 train_time:190270ms step_avg:158.43ms
step:1212/1480 train_time:190439ms step_avg:158.44ms
step:1213/1480 train_time:190613ms step_avg:158.45ms
step:1214/1480 train_time:190791ms step_avg:158.46ms
step:1215/1480 train_time:190964ms step_avg:158.48ms
step:1216/1480 train_time:191134ms step_avg:158.49ms
step:1217/1480 train_time:191309ms step_avg:158.50ms
step:1218/1480 train_time:191480ms step_avg:158.51ms
step:1219/1480 train_time:191658ms step_avg:158.53ms
step:1220/1480 train_time:191828ms step_avg:158.54ms
step:1221/1480 train_time:191998ms step_avg:158.55ms
step:1222/1480 train_time:192166ms step_avg:158.55ms
step:1223/1480 train_time:192336ms step_avg:158.56ms
step:1224/1480 train_time:192513ms step_avg:158.58ms
step:1225/1480 train_time:192685ms step_avg:158.59ms
step:1226/1480 train_time:192857ms step_avg:158.60ms
step:1227/1480 train_time:193031ms step_avg:158.61ms
step:1228/1480 train_time:193201ms step_avg:158.62ms
step:1229/1480 train_time:193375ms step_avg:158.63ms
step:1230/1480 train_time:193555ms step_avg:158.65ms
step:1231/1480 train_time:193730ms step_avg:158.67ms
step:1232/1480 train_time:193905ms step_avg:158.68ms
step:1233/1480 train_time:194074ms step_avg:158.69ms
step:1234/1480 train_time:194245ms step_avg:158.70ms
step:1235/1480 train_time:194416ms step_avg:158.71ms
step:1236/1480 train_time:194585ms step_avg:158.72ms
step:1237/1480 train_time:194756ms step_avg:158.73ms
step:1238/1480 train_time:194942ms step_avg:158.75ms
step:1239/1480 train_time:195114ms step_avg:158.76ms
step:1240/1480 train_time:195285ms step_avg:158.77ms
step:1241/1480 train_time:195458ms step_avg:158.78ms
step:1242/1480 train_time:195628ms step_avg:158.79ms
step:1243/1480 train_time:195801ms step_avg:158.80ms
step:1244/1480 train_time:195969ms step_avg:158.81ms
step:1245/1480 train_time:196137ms step_avg:158.82ms
step:1246/1480 train_time:196307ms step_avg:158.82ms
step:1247/1480 train_time:196475ms step_avg:158.83ms
step:1248/1480 train_time:196645ms step_avg:158.84ms
step:1249/1480 train_time:196813ms step_avg:158.85ms
step:1250/1480 train_time:196981ms step_avg:158.86ms
step:1250/1480 val_loss:3.3360 train_time:197055ms step_avg:158.92ms
step:1251/1480 train_time:197162ms step_avg:158.87ms
step:1252/1480 train_time:197331ms step_avg:158.88ms
step:1253/1480 train_time:197499ms step_avg:158.89ms
step:1254/1480 train_time:197672ms step_avg:158.90ms
step:1255/1480 train_time:197858ms step_avg:158.92ms
step:1256/1480 train_time:198031ms step_avg:158.93ms
step:1257/1480 train_time:198202ms step_avg:158.94ms
step:1258/1480 train_time:198377ms step_avg:158.96ms
step:1259/1480 train_time:198549ms step_avg:158.97ms
step:1260/1480 train_time:198715ms step_avg:158.97ms
step:1261/1480 train_time:198889ms step_avg:158.98ms
step:1262/1480 train_time:199065ms step_avg:159.00ms
step:1263/1480 train_time:199238ms step_avg:159.01ms
step:1264/1480 train_time:199406ms step_avg:159.02ms
step:1265/1480 train_time:199572ms step_avg:159.02ms
step:1266/1480 train_time:199744ms step_avg:159.03ms
step:1267/1480 train_time:199914ms step_avg:159.04ms
step:1268/1480 train_time:200087ms step_avg:159.05ms
step:1269/1480 train_time:200263ms step_avg:159.07ms
step:1270/1480 train_time:200433ms step_avg:159.07ms
step:1271/1480 train_time:200604ms step_avg:159.08ms
step:1272/1480 train_time:200770ms step_avg:159.09ms
step:1273/1480 train_time:200940ms step_avg:159.10ms
step:1274/1480 train_time:201113ms step_avg:159.11ms
step:1275/1480 train_time:201280ms step_avg:159.11ms
step:1276/1480 train_time:201448ms step_avg:159.12ms
step:1277/1480 train_time:201619ms step_avg:159.13ms
step:1278/1480 train_time:201789ms step_avg:159.14ms
step:1279/1480 train_time:201961ms step_avg:159.15ms
step:1280/1480 train_time:202139ms step_avg:159.16ms
step:1281/1480 train_time:202308ms step_avg:159.17ms
step:1282/1480 train_time:202474ms step_avg:159.18ms
step:1283/1480 train_time:202644ms step_avg:159.19ms
step:1284/1480 train_time:202813ms step_avg:159.19ms
step:1285/1480 train_time:202982ms step_avg:159.20ms
step:1286/1480 train_time:203152ms step_avg:159.21ms
step:1287/1480 train_time:203323ms step_avg:159.22ms
step:1288/1480 train_time:203495ms step_avg:159.23ms
step:1289/1480 train_time:203677ms step_avg:159.25ms
step:1290/1480 train_time:203857ms step_avg:159.26ms
step:1291/1480 train_time:204029ms step_avg:159.27ms
step:1292/1480 train_time:204202ms step_avg:159.28ms
step:1293/1480 train_time:204378ms step_avg:159.30ms
step:1294/1480 train_time:204550ms step_avg:159.31ms
step:1295/1480 train_time:204722ms step_avg:159.32ms
step:1296/1480 train_time:204896ms step_avg:159.33ms
step:1297/1480 train_time:205069ms step_avg:159.34ms
step:1298/1480 train_time:205239ms step_avg:159.35ms
step:1299/1480 train_time:205410ms step_avg:159.36ms
step:1300/1480 train_time:205577ms step_avg:159.36ms
step:1301/1480 train_time:205745ms step_avg:159.37ms
step:1302/1480 train_time:205920ms step_avg:159.38ms
step:1303/1480 train_time:206099ms step_avg:159.40ms
step:1304/1480 train_time:206272ms step_avg:159.41ms
step:1305/1480 train_time:206441ms step_avg:159.41ms
step:1306/1480 train_time:206615ms step_avg:159.42ms
step:1307/1480 train_time:206782ms step_avg:159.43ms
step:1308/1480 train_time:206952ms step_avg:159.44ms
step:1309/1480 train_time:207125ms step_avg:159.45ms
step:1310/1480 train_time:207293ms step_avg:159.46ms
step:1311/1480 train_time:207463ms step_avg:159.46ms
step:1312/1480 train_time:207635ms step_avg:159.47ms
step:1313/1480 train_time:207805ms step_avg:159.48ms
step:1314/1480 train_time:207978ms step_avg:159.49ms
step:1315/1480 train_time:208149ms step_avg:159.50ms
step:1316/1480 train_time:208315ms step_avg:159.51ms
step:1317/1480 train_time:208487ms step_avg:159.52ms
step:1318/1480 train_time:208668ms step_avg:159.53ms
step:1319/1480 train_time:208844ms step_avg:159.54ms
step:1320/1480 train_time:209020ms step_avg:159.56ms
step:1321/1480 train_time:209193ms step_avg:159.57ms
step:1322/1480 train_time:209373ms step_avg:159.58ms
step:1323/1480 train_time:209546ms step_avg:159.59ms
step:1324/1480 train_time:209722ms step_avg:159.61ms
step:1325/1480 train_time:209902ms step_avg:159.62ms
step:1326/1480 train_time:210078ms step_avg:159.63ms
step:1327/1480 train_time:210249ms step_avg:159.64ms
step:1328/1480 train_time:210418ms step_avg:159.65ms
step:1329/1480 train_time:210614ms step_avg:159.68ms
step:1330/1480 train_time:210793ms step_avg:159.69ms
step:1331/1480 train_time:210963ms step_avg:159.70ms
step:1332/1480 train_time:211138ms step_avg:159.71ms
step:1333/1480 train_time:211315ms step_avg:159.72ms
step:1334/1480 train_time:211488ms step_avg:159.73ms
step:1335/1480 train_time:211658ms step_avg:159.74ms
step:1336/1480 train_time:211843ms step_avg:159.76ms
step:1337/1480 train_time:212018ms step_avg:159.77ms
step:1338/1480 train_time:212192ms step_avg:159.78ms
step:1339/1480 train_time:212365ms step_avg:159.79ms
step:1340/1480 train_time:212538ms step_avg:159.80ms
step:1341/1480 train_time:212707ms step_avg:159.81ms
step:1342/1480 train_time:212881ms step_avg:159.82ms
step:1343/1480 train_time:213053ms step_avg:159.83ms
step:1344/1480 train_time:213224ms step_avg:159.84ms
step:1345/1480 train_time:213403ms step_avg:159.85ms
step:1346/1480 train_time:213571ms step_avg:159.86ms
step:1347/1480 train_time:213740ms step_avg:159.87ms
step:1348/1480 train_time:213911ms step_avg:159.87ms
step:1349/1480 train_time:214080ms step_avg:159.88ms
step:1350/1480 train_time:214254ms step_avg:159.89ms
step:1351/1480 train_time:214425ms step_avg:159.90ms
step:1352/1480 train_time:214596ms step_avg:159.91ms
step:1353/1480 train_time:214772ms step_avg:159.92ms
step:1354/1480 train_time:214944ms step_avg:159.93ms
step:1355/1480 train_time:215110ms step_avg:159.93ms
step:1356/1480 train_time:215283ms step_avg:159.94ms
step:1357/1480 train_time:215456ms step_avg:159.95ms
step:1358/1480 train_time:215628ms step_avg:159.96ms
step:1359/1480 train_time:215800ms step_avg:159.97ms
step:1360/1480 train_time:215974ms step_avg:159.98ms
step:1361/1480 train_time:216152ms step_avg:159.99ms
step:1362/1480 train_time:216329ms step_avg:160.01ms
step:1363/1480 train_time:216507ms step_avg:160.02ms
step:1364/1480 train_time:216675ms step_avg:160.03ms
step:1365/1480 train_time:216843ms step_avg:160.03ms
step:1366/1480 train_time:217016ms step_avg:160.04ms
step:1367/1480 train_time:217188ms step_avg:160.05ms
step:1368/1480 train_time:217359ms step_avg:160.06ms
step:1369/1480 train_time:217540ms step_avg:160.07ms
step:1370/1480 train_time:217717ms step_avg:160.09ms
step:1371/1480 train_time:217891ms step_avg:160.10ms
step:1372/1480 train_time:218069ms step_avg:160.11ms
step:1373/1480 train_time:218237ms step_avg:160.12ms
step:1374/1480 train_time:218415ms step_avg:160.13ms
step:1375/1480 train_time:218587ms step_avg:160.14ms
step:1375/1480 val_loss:3.2974 train_time:218655ms step_avg:160.19ms
step:1376/1480 train_time:218761ms step_avg:160.15ms
step:1377/1480 train_time:218934ms step_avg:160.16ms
step:1378/1480 train_time:219105ms step_avg:160.16ms
step:1379/1480 train_time:219281ms step_avg:160.18ms
step:1380/1480 train_time:219454ms step_avg:160.19ms
step:1381/1480 train_time:219632ms step_avg:160.20ms
step:1382/1480 train_time:219805ms step_avg:160.21ms
step:1383/1480 train_time:219976ms step_avg:160.22ms
step:1384/1480 train_time:220154ms step_avg:160.23ms
step:1385/1480 train_time:220320ms step_avg:160.23ms
step:1386/1480 train_time:220490ms step_avg:160.24ms
step:1387/1480 train_time:220662ms step_avg:160.25ms
step:1388/1480 train_time:220833ms step_avg:160.26ms
step:1389/1480 train_time:221008ms step_avg:160.27ms
step:1390/1480 train_time:221176ms step_avg:160.27ms
step:1391/1480 train_time:221347ms step_avg:160.28ms
step:1392/1480 train_time:221520ms step_avg:160.29ms
step:1393/1480 train_time:221692ms step_avg:160.30ms
step:1394/1480 train_time:221863ms step_avg:160.31ms
step:1395/1480 train_time:222031ms step_avg:160.31ms
step:1396/1480 train_time:222201ms step_avg:160.32ms
step:1397/1480 train_time:222368ms step_avg:160.32ms
step:1398/1480 train_time:222535ms step_avg:160.33ms
step:1399/1480 train_time:222706ms step_avg:160.34ms
step:1400/1480 train_time:222884ms step_avg:160.35ms
step:1401/1480 train_time:223050ms step_avg:160.35ms
step:1402/1480 train_time:223222ms step_avg:160.36ms
step:1403/1480 train_time:223399ms step_avg:160.37ms
step:1404/1480 train_time:223570ms step_avg:160.38ms
step:1405/1480 train_time:223745ms step_avg:160.39ms
step:1406/1480 train_time:223920ms step_avg:160.40ms
step:1407/1480 train_time:224088ms step_avg:160.41ms
step:1408/1480 train_time:224256ms step_avg:160.41ms
step:1409/1480 train_time:224439ms step_avg:160.43ms
step:1410/1480 train_time:224608ms step_avg:160.43ms
step:1411/1480 train_time:224776ms step_avg:160.44ms
step:1412/1480 train_time:224946ms step_avg:160.45ms
step:1413/1480 train_time:225117ms step_avg:160.45ms
step:1414/1480 train_time:225288ms step_avg:160.46ms
step:1415/1480 train_time:225462ms step_avg:160.47ms
step:1416/1480 train_time:225649ms step_avg:160.49ms
step:1417/1480 train_time:225825ms step_avg:160.50ms
step:1418/1480 train_time:225996ms step_avg:160.51ms
step:1419/1480 train_time:226169ms step_avg:160.52ms
step:1420/1480 train_time:226343ms step_avg:160.53ms
step:1421/1480 train_time:226517ms step_avg:160.54ms
step:1422/1480 train_time:226690ms step_avg:160.55ms
step:1423/1480 train_time:226859ms step_avg:160.55ms
step:1424/1480 train_time:227036ms step_avg:160.56ms
step:1425/1480 train_time:227215ms step_avg:160.58ms
step:1426/1480 train_time:227387ms step_avg:160.58ms
step:1427/1480 train_time:227563ms step_avg:160.59ms
step:1428/1480 train_time:227735ms step_avg:160.60ms
step:1429/1480 train_time:227905ms step_avg:160.61ms
step:1430/1480 train_time:228079ms step_avg:160.62ms
step:1431/1480 train_time:228253ms step_avg:160.63ms
step:1432/1480 train_time:228430ms step_avg:160.64ms
step:1433/1480 train_time:228611ms step_avg:160.65ms
step:1434/1480 train_time:228790ms step_avg:160.67ms
step:1435/1480 train_time:228967ms step_avg:160.68ms
step:1436/1480 train_time:229140ms step_avg:160.69ms
step:1437/1480 train_time:229310ms step_avg:160.69ms
step:1438/1480 train_time:229479ms step_avg:160.70ms
step:1439/1480 train_time:229652ms step_avg:160.71ms
step:1440/1480 train_time:229822ms step_avg:160.71ms
step:1441/1480 train_time:229992ms step_avg:160.72ms
step:1442/1480 train_time:230170ms step_avg:160.73ms
step:1443/1480 train_time:230357ms step_avg:160.75ms
step:1444/1480 train_time:230529ms step_avg:160.76ms
step:1445/1480 train_time:230702ms step_avg:160.77ms
step:1446/1480 train_time:230877ms step_avg:160.78ms
step:1447/1480 train_time:231053ms step_avg:160.79ms
step:1448/1480 train_time:231226ms step_avg:160.80ms
step:1449/1480 train_time:231400ms step_avg:160.81ms
step:1450/1480 train_time:231572ms step_avg:160.81ms
step:1451/1480 train_time:231744ms step_avg:160.82ms
step:1452/1480 train_time:231917ms step_avg:160.83ms
step:1453/1480 train_time:232088ms step_avg:160.84ms
step:1454/1480 train_time:232262ms step_avg:160.85ms
step:1455/1480 train_time:232440ms step_avg:160.86ms
step:1456/1480 train_time:232613ms step_avg:160.87ms
step:1457/1480 train_time:232786ms step_avg:160.87ms
step:1458/1480 train_time:232956ms step_avg:160.88ms
step:1459/1480 train_time:233133ms step_avg:160.89ms
step:1460/1480 train_time:233306ms step_avg:160.90ms
step:1461/1480 train_time:233481ms step_avg:160.91ms
step:1462/1480 train_time:233652ms step_avg:160.92ms
step:1463/1480 train_time:233830ms step_avg:160.93ms
step:1464/1480 train_time:234006ms step_avg:160.94ms
step:1465/1480 train_time:234177ms step_avg:160.95ms
step:1466/1480 train_time:234348ms step_avg:160.95ms
step:1467/1480 train_time:234522ms step_avg:160.96ms
step:1468/1480 train_time:234692ms step_avg:160.97ms
step:1469/1480 train_time:234864ms step_avg:160.98ms
step:1470/1480 train_time:235044ms step_avg:160.99ms
step:1471/1480 train_time:235231ms step_avg:161.01ms
step:1472/1480 train_time:235411ms step_avg:161.02ms
step:1473/1480 train_time:235582ms step_avg:161.03ms
step:1474/1480 train_time:235760ms step_avg:161.04ms
step:1475/1480 train_time:235939ms step_avg:161.05ms
step:1476/1480 train_time:236111ms step_avg:161.06ms
step:1477/1480 train_time:236292ms step_avg:161.07ms
step:1478/1480 train_time:236474ms step_avg:161.09ms
step:1479/1480 train_time:236649ms step_avg:161.10ms
step:1480/1480 train_time:236822ms step_avg:161.10ms
step:1480/1480 val_loss:3.2783 train_time:236894ms step_avg:161.15ms