e3319113-ec3a-457a-975c-e89dcddd15f2.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
from dataclasses import dataclass
from pathlib import Path

import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
# Use of FlexAttention contributed by @KoszarskyB
from torch.nn.attention.flex_attention import BlockMask, flex_attention

# -----------------------------------------------------------------------------
# Muon optimizer

def zeropower_via_svd(G, steps=None):
    U, S, V = G.svd()
    return U @ V.T

@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert len(G.shape) == 2
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    X /= (X.norm() + eps) # ensure top singular value <= 1
    if G.size(0) > G.size(1):
        X = X.T
    for _ in range(steps):
        A = X @ X.T
        B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    if G.size(0) > G.size(1):
        X = X.T
    return X

zeropower_backends = dict(svd=zeropower_via_svd, newtonschulz5=zeropower_via_newtonschulz5)

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven't tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        backend: The chosen backend for the orthogonalization step. (recommended: 'newtonschulz5')
        backend_steps: The number of iteration steps to use in the backend, if it is iterative.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True,
                 backend='newtonschulz5', backend_steps=5):
        self.num_process = int(os.environ['WORLD_SIZE'])
        self.rank = int(os.environ["RANK"])
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, backend=backend, backend_steps=backend_steps)
        params: "list[torch.Tensor]" = list(params)
        assert all(isinstance(p, torch.Tensor) for p in params)
        sizes = {p.numel() for p in params}
        param_groups = [
            {
                "params": [p for p in params if p.numel() == size],
                "update_buffer": [
                    torch.empty(size, device="cuda", dtype=torch.bfloat16)
                    for _ in range(self.num_process)
                ],
            }
            for size in sizes
        ]
        super().__init__(param_groups, defaults)

    def step(self):
        for group in self.param_groups:
            lr: float = group["lr"]
            momentum: float = group["momentum"]
            nesterov: bool = group["nesterov"]
            zeropower_backend = zeropower_backends[group["backend"]]
            backend_steps: int = group["backend_steps"]
            update_buffers: "list[torch.Tensor]" = group["update_buffer"]
            # generate weight updates in distributed fashion
            params: "list[torch.Tensor]" = group["params"]
            assert len(params) % self.num_process == 0
            handle = None
            params_world = None
            def update_prev():
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffers):
                    p_world.data.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.num_process]:
                p = params[base_i + self.rank]
                g = p.grad
                assert g is not None
                state = self.state[p] 
                if "momentum_buffer" not in state:
                    state["momentum_buffer"] = torch.zeros_like(g)
                buf: torch.Tensor = state["momentum_buffer"]
                buf.lerp_(g, 1 - momentum)
                g = g.lerp_(buf, momentum) if nesterov else buf
                g = zeropower_backend(g, steps=backend_steps).flatten()
                update_prev()
                handle = dist.all_gather(update_buffers, g, async_op=True)
                params_world = params[base_i : base_i + self.num_process]
            update_prev()


# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):

    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=False)

    def forward(self, x):
        return F.linear(x, self.weight.to(x.dtype))

class Rotary(torch.nn.Module):

    def __init__(self, dim, base=10000):
        super().__init__()
        self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None

    def forward(self, x):
        seq_len = x.shape[1]
        if seq_len != self.seq_len_cached:
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.outer(t, self.inv_freq)
            self.seq_len_cached = seq_len
            self.cos_cached = freqs.cos()
            self.sin_cached = freqs.sin()
        cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
        # apply_rotary_emb(x, cos, sin)
        x1, x2 = x.chunk(2, dim=3)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x)

class CausalSelfAttention(nn.Module):

    def __init__(self, dim, n_head):
        super().__init__()
        assert dim % n_head == 0
        self.n_head = n_head
        self.c_q = CastedLinear(dim, dim)
        self.c_k = CastedLinear(dim, dim)
        self.c_v = CastedLinear(dim, dim)
        # value residual lambda
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5])) # @Grad62304977
        # rotary embeddings
        self.rotary = Rotary(dim // n_head) # dim // n_head = head_dim
        # output projection
        self.c_proj = CastedLinear(dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor, vi: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q: torch.Tensor = self.c_q(x).view(B, T, self.n_head, -1)
        k: torch.Tensor = self.c_k(x).view(B, T, self.n_head, -1)
        v: torch.Tensor = self.c_v(x).view(B, T, self.n_head, -1)
        v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @Grad62304977
        q, k = norm(q), norm(k) # QK norm suggested by @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask)
        y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):

    def __init__(self, dim: int):
        super().__init__()
        self.c_fc   = CastedLinear(dim, 4 * dim)
        self.c_proj = CastedLinear(4 * dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):

    def __init__(self, config):
        super().__init__()
        self.attn = CausalSelfAttention(config.n_embd, config.n_head)
        self.mlp = MLP(config.n_embd)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x: torch.Tensor, vi: torch.Tensor, x0: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        x = x + self.attn(norm(x), vi, block_mask)
        x = x + self.mlp(norm(x))
        return x

# -----------------------------------------------------------------------------
# The main GPT-2 model

@dataclass
class GPTConfig:
    vocab_size : int = 50304
    n_layer : int = 12
    n_head : int = 6 # head dim 128 suggested by @Grad62304977
    n_embd : int = 768
    lm_head_softcap : int = 30

class GPT(nn.Module):

    def __init__(self, config: GPTConfig):
        super().__init__()
        self.n_layer = config.n_layer
        self.lm_head_softcap = config.lm_head_softcap

        # U-net design by @brendanh0gan
        self.num_encoder_layers = config.n_layer // 2 # Half of the layers for encoder
        self.num_decoder_layers = config.n_layer - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))

        self.transformer = nn.ModuleDict(dict(
            wte = nn.Embedding(config.vocab_size, config.n_embd),
            # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
            # U-net structure on token value embeddings by @leloykun
            vte = nn.Embedding(config.vocab_size, config.n_embd*self.num_encoder_layers),
            h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
        ))
        self.lm_head = CastedLinear(config.n_embd, config.vocab_size)
        self.lm_head.weight.data.zero_() # @Grad62304977

    def forward(self, idx: torch.Tensor, target: torch.Tensor, sliding_window: torch.Tensor) -> torch.Tensor:
        BLOCK_SIZE = 128
        assert idx.ndim == 1
        docs = (idx == 50256).cumsum(0)
        docs_low = docs.reshape(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.reshape(-1, BLOCK_SIZE)[:, -1].contiguous()
        def document_sliding_window_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            window_mask = q_idx - kv_idx < sliding_window
            return causal_mask & document_mask & window_mask

        S = len(idx)
        def create_sliding_window_causal_mask(S: int, sliding_window: torch.Tensor):
            kv_idx = block_idx = torch.arange(S // BLOCK_SIZE, dtype=torch.int32, device="cuda")
            q_idx = block_idx[:, None]
            causal_mask = q_idx >= kv_idx
            document_mask = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
            window_mask = q_idx - kv_idx < ((sliding_window + BLOCK_SIZE - 1) // BLOCK_SIZE)
            dense_mask = causal_mask & document_mask & window_mask
            dense_mask = dense_mask.to(torch.int32)
            num_blocks = dense_mask.sum(dim=-1).to(torch.int32)
            indices = torch.argsort(dense_mask, dim=-1, descending=True, stable=True).to(torch.int32)
            num_blocks = num_blocks[None, None, :].contiguous()
            indices = indices[None, None, :].contiguous()
            return BlockMask.from_kv_blocks(num_blocks, indices, BLOCK_SIZE=BLOCK_SIZE, mask_mod=document_sliding_window_causal)
        block_mask = create_sliding_window_causal_mask(S, sliding_window)

        # forward the GPT model itself
        x = self.transformer.wte(idx[None]) # token embeddings of shape (b, t, n_embd)
        x = norm(x) # @Grad62304977
        x0 = x
        vi = self.transformer.vte(idx[None]).chunk(self.num_encoder_layers, dim=-1)

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        for i in range(self.num_encoder_layers):
            x = self.transformer.h[i](x, vi[i], x0, block_mask)
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            # U-net structure on token value embeddings by @leloykun
            x = self.transformer.h[self.num_encoder_layers + i](x, vi[self.num_encoder_layers-1-i], x0, block_mask)

        x = norm(x)
        logits = self.lm_head(x)
        logits = self.lm_head_softcap * torch.tanh(logits / self.lm_head_softcap) # @Grad62304977
        logits = logits.float()
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), target.view(-1))
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _peek_data_shard(file: Path):
    # only reads the header, returns header data
    # header is 256 int32
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    return int(header[2]) # number of tokens (claimed)

def _load_data_shard(file: Path, ntok: int):
    with file.open("rb") as f:
        tokens = torch.empty(ntok, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy())
        assert nbytes == 2 * ntok, "number of tokens read does not match header?"
    return tokens

class DistributedDataLoader:
    def __init__(self, filename_pattern, T, process_rank, num_processes):
        self.process_rank = process_rank
        self.num_processes = num_processes
        self.T = T

        # glob files that match the pattern
        self.files = sorted(Path.cwd().glob(filename_pattern))
        assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"

        # load and validate all data shards, count number of tokens in total
        self.ntoks = [_peek_data_shard(file) for file in self.files]
        assert min(self.ntoks) >= num_processes * T + 1
        self.ntok_total = sum(self.ntoks)

        self.reset()

    def reset(self):
        self.current_shard = -1
        self.advance()

    def advance(self): # advance to next data shard
        self.current_shard = (self.current_shard + 1) % len(self.files)
        self.current_position = self.process_rank * self.T
        self.tokens = _load_data_shard(self.files[self.current_shard], self.ntoks[self.current_shard])

    def next_batch(self):
        batch_size = self.T * self.num_processes
        buf = self.tokens[self.current_position:self.current_position+self.T+1]
        # host side async is sufficient;
        # no performance improvement was observed when introducing a separate stream.
        x = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
        y = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
        # advance current position and load next shard if necessary
        self.current_position += batch_size
        if self.current_position + batch_size + 1 >= len(self.tokens):
            self.advance()
        return x, y

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data hyperparams
    input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
    input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
    # optimization hyperparams
    batch_size : int = 8 # batch size, in sequences, across all devices
    sequence_length : int = 64*1024 # sequence length, in tokens
    num_iterations : int = 1480 # number of iterations to run
    warmup_iters : int = 0
    cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
    weight_decay : float = 0
    # evaluation and logging hyperparams
    val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
    val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()

# set up DDP (distributed data parallel). torchrun sets this env variable
assert torch.cuda.is_available()
dist.init_process_group(backend='nccl')
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
device = f'cuda:{ddp_local_rank}'
torch.cuda.set_device(device)
print(f"using device: {device}")
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = str(uuid.uuid4())
    logdir = 'logs/%s/' % run_id
    # os.makedirs(logdir, exist_ok=True)
    logfile = 'logs/%s.txt' % run_id
    # create the log file
    with open(logfile, "w") as f:
        # begin the log by printing this file (the Python code)
        f.write(code)
        f.write('='*100 + '\n')
def print0(s, logonly=False):
    if master_process:
        with open(logfile, "a") as f:
            if not logonly:
                print(s)
            f.write(s+'\n')
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)

# convenience variables
T = args.sequence_length
# calculate the number of steps to take in the val loop.
assert args.val_tokens % (T * ddp_world_size) == 0
val_steps = args.val_tokens // (T * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size
assert train_accumulation_steps == 1

# load tokens
train_loader = DistributedDataLoader(args.input_bin, T, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, T, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.ntok_total} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.ntok_total} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
x, y = train_loader.next_batch()

# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, n_layer=12, n_head=6, n_embd=768))
model = model.cuda().bfloat16()
for m in model.modules():
    if isinstance(m, CastedLinear):
        m.float()
if hasattr(config, "coordinate_descent_tuning"):
    config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank])
raw_model = model.module # always contains the "raw" unwrapped model

# init the optimizer(s)
optimizer1 = torch.optim.Adam([raw_model.transformer.wte.weight, raw_model.transformer.vte.weight], lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.transformer.h.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
    assert it <= args.num_iterations
    # 1) linear warmup for warmup_iters steps
    if it < args.warmup_iters:
        return (it+1) / args.warmup_iters
    # 2) constant lr for a while
    elif it < args.num_iterations - args.cooldown_iters:
        return 1.0
    # 3) linear cooldown
    else:
        decay_ratio = (args.num_iterations - it) / args.cooldown_iters
        return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]

sliding_window_size = torch.tensor(64, dtype=torch.int32, device="cuda")
sw_size_prev = 64
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
    last_step = (step == args.num_iterations)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Set the sliding window size for the current step, in chunks of 64. By @fernbear.bsky.social
    sw_size =  64 * int((64 + (1792 - 64) * step / args.num_iterations) // 64)
    if sw_size != sw_size_prev:
        sliding_window_size.copy_(sw_size, non_blocking=True)
        sw_size_prev = sw_size

    # once in a while evaluate the validation dataset
    if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # run validation batches
        model.eval()
        val_loader.reset()
        val_loss = 0.0
        for _ in range(val_steps):
            with torch.no_grad():
                x_val, y_val = val_loader.next_batch()
                val_loss += model(x_val, y_val, sliding_window=sliding_window_size)
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        val_loss /= val_steps
        # log val loss to console and to logfile
        print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # save the state of the training process
        log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
        # torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    # bit confusing: we want to make sure to eval on 0th iteration
    # but also after the very last iteration. so we loop for step <= num_iterations
    # instead of just < num_iterations (one extra due to <=), only to do
    # the validation/sampling one last time, and then we break right here as we're done.
    if last_step:
        break

    # --------------- TRAINING SECTION BEGIN -----------------
    model.train()
    loss = model(x, y, sliding_window=sliding_window_size)
    loss.backward()
    del loss
    # advance the dataset for the next batch
    x, y = train_loader.next_batch()
    # momentum warmup for Muon
    frac = min(step/300, 1)
    for group in optimizer3.param_groups:
        group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # --------------- TRAINING SECTION END -------------------
    # everything that follows now is just diagnostics, prints, logging, etc.
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")

if master_process:
    print(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")

# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()
====================================================================================================
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Sun Dec  8 07:50:12 2024       
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.6     |
|-----------------------------------------+----------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
|                                         |                      |               MIG M. |
|=========================================+======================+======================|
|   0  NVIDIA H100 80GB HBM3          On  | 00000000:65:02.0 Off |                    0 |
| N/A   37C    P0              74W / 700W |      7MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  | 00000000:67:02.0 Off |                    0 |
| N/A   46C    P0             124W / 700W |    533MiB / 81559MiB |      1%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  | 00000000:69:02.0 Off |                    0 |
| N/A   46C    P0             124W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  | 00000000:6B:02.0 Off |                    0 |
| N/A   39C    P0             118W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  | 00000000:6F:02.0 Off |                    0 |
| N/A   39C    P0             108W / 700W |     45MiB / 81559MiB |      1%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  | 00000000:71:02.0 Off |                    0 |
| N/A   46C    P0              98W / 700W |     27MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  | 00000000:73:02.0 Off |                    0 |
| N/A   46C    P0             127W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  | 00000000:75:02.0 Off |                    0 |
| N/A   39C    P0             124W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
                                                                                         
+---------------------------------------------------------------------------------------+
| Processes:                                                                            |
|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
|        ID   ID                                                             Usage      |
|=======================================================================================|
+---------------------------------------------------------------------------------------+

====================================================================================================
Training DataLoader: total number of tokens: 3200000000 across 32 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:23632ms step_avg:nanms
step:2/1480 train_time:23725ms step_avg:nanms
step:3/1480 train_time:23862ms step_avg:nanms
step:4/1480 train_time:24003ms step_avg:nanms
step:5/1480 train_time:24144ms step_avg:nanms
step:6/1480 train_time:24284ms step_avg:nanms
step:7/1480 train_time:24426ms step_avg:nanms
step:8/1480 train_time:24570ms step_avg:nanms
step:9/1480 train_time:24717ms step_avg:nanms
step:10/1480 train_time:24859ms step_avg:nanms
step:11/1480 train_time:142ms step_avg:nanms
step:12/1480 train_time:283ms step_avg:nanms
step:13/1480 train_time:423ms step_avg:141.07ms
step:14/1480 train_time:565ms step_avg:141.17ms
step:15/1480 train_time:706ms step_avg:141.26ms
step:16/1480 train_time:851ms step_avg:141.78ms
step:17/1480 train_time:995ms step_avg:142.14ms
step:18/1480 train_time:1140ms step_avg:142.53ms
step:19/1480 train_time:1283ms step_avg:142.52ms
step:20/1480 train_time:1424ms step_avg:142.36ms
step:21/1480 train_time:1565ms step_avg:142.27ms
step:22/1480 train_time:1707ms step_avg:142.29ms
step:23/1480 train_time:1850ms step_avg:142.28ms
step:24/1480 train_time:1993ms step_avg:142.35ms
step:25/1480 train_time:2136ms step_avg:142.39ms
step:26/1480 train_time:2280ms step_avg:142.51ms
step:27/1480 train_time:2423ms step_avg:142.51ms
step:28/1480 train_time:2564ms step_avg:142.47ms
step:29/1480 train_time:2706ms step_avg:142.43ms
step:30/1480 train_time:2848ms step_avg:142.38ms
step:31/1480 train_time:2989ms step_avg:142.33ms
step:32/1480 train_time:3131ms step_avg:142.33ms
step:33/1480 train_time:3275ms step_avg:142.40ms
step:34/1480 train_time:3419ms step_avg:142.44ms
step:35/1480 train_time:3561ms step_avg:142.45ms
step:36/1480 train_time:3703ms step_avg:142.44ms
step:37/1480 train_time:3845ms step_avg:142.41ms
step:38/1480 train_time:3986ms step_avg:142.35ms
step:39/1480 train_time:4127ms step_avg:142.32ms
step:40/1480 train_time:4271ms step_avg:142.35ms
step:41/1480 train_time:4414ms step_avg:142.39ms
step:42/1480 train_time:4561ms step_avg:142.53ms
step:43/1480 train_time:4701ms step_avg:142.44ms
step:44/1480 train_time:4843ms step_avg:142.43ms
step:45/1480 train_time:4985ms step_avg:142.41ms
step:46/1480 train_time:5126ms step_avg:142.39ms
step:47/1480 train_time:5268ms step_avg:142.38ms
step:48/1480 train_time:5410ms step_avg:142.36ms
step:49/1480 train_time:5554ms step_avg:142.41ms
step:50/1480 train_time:5697ms step_avg:142.42ms
step:51/1480 train_time:5840ms step_avg:142.44ms
step:52/1480 train_time:5983ms step_avg:142.45ms
step:53/1480 train_time:6124ms step_avg:142.42ms
step:54/1480 train_time:6266ms step_avg:142.41ms
step:55/1480 train_time:6409ms step_avg:142.42ms
step:56/1480 train_time:6552ms step_avg:142.43ms
step:57/1480 train_time:6694ms step_avg:142.42ms
step:58/1480 train_time:6837ms step_avg:142.43ms
step:59/1480 train_time:6981ms step_avg:142.47ms
step:60/1480 train_time:7123ms step_avg:142.46ms
step:61/1480 train_time:7266ms step_avg:142.47ms
step:62/1480 train_time:7408ms step_avg:142.45ms
step:63/1480 train_time:7550ms step_avg:142.46ms
step:64/1480 train_time:7692ms step_avg:142.44ms
step:65/1480 train_time:7835ms step_avg:142.45ms
step:66/1480 train_time:7978ms step_avg:142.47ms
step:67/1480 train_time:8120ms step_avg:142.46ms
step:68/1480 train_time:8263ms step_avg:142.47ms
step:69/1480 train_time:8405ms step_avg:142.46ms
step:70/1480 train_time:8549ms step_avg:142.48ms
step:71/1480 train_time:8692ms step_avg:142.49ms
step:72/1480 train_time:8835ms step_avg:142.50ms
step:73/1480 train_time:8977ms step_avg:142.50ms
step:74/1480 train_time:9121ms step_avg:142.51ms
step:75/1480 train_time:9263ms step_avg:142.51ms
step:76/1480 train_time:9406ms step_avg:142.52ms
step:77/1480 train_time:9547ms step_avg:142.50ms
step:78/1480 train_time:9688ms step_avg:142.47ms
step:79/1480 train_time:9829ms step_avg:142.45ms
step:80/1480 train_time:9972ms step_avg:142.46ms
step:81/1480 train_time:10117ms step_avg:142.49ms
step:82/1480 train_time:10260ms step_avg:142.50ms
step:83/1480 train_time:10403ms step_avg:142.51ms
step:84/1480 train_time:10546ms step_avg:142.51ms
step:85/1480 train_time:10686ms step_avg:142.49ms
step:86/1480 train_time:10828ms step_avg:142.47ms
step:87/1480 train_time:10971ms step_avg:142.48ms
step:88/1480 train_time:11113ms step_avg:142.48ms
step:89/1480 train_time:11255ms step_avg:142.47ms
step:90/1480 train_time:11399ms step_avg:142.48ms
step:91/1480 train_time:11541ms step_avg:142.48ms
step:92/1480 train_time:11683ms step_avg:142.47ms
step:93/1480 train_time:11824ms step_avg:142.46ms
step:94/1480 train_time:11966ms step_avg:142.45ms
step:95/1480 train_time:12108ms step_avg:142.45ms
step:96/1480 train_time:12250ms step_avg:142.44ms
step:97/1480 train_time:12391ms step_avg:142.42ms
step:98/1480 train_time:12535ms step_avg:142.44ms
step:99/1480 train_time:12677ms step_avg:142.44ms
step:100/1480 train_time:12821ms step_avg:142.45ms
step:101/1480 train_time:12963ms step_avg:142.45ms
step:102/1480 train_time:13105ms step_avg:142.44ms
step:103/1480 train_time:13246ms step_avg:142.43ms
step:104/1480 train_time:13387ms step_avg:142.41ms
step:105/1480 train_time:13528ms step_avg:142.40ms
step:106/1480 train_time:13670ms step_avg:142.40ms
step:107/1480 train_time:13814ms step_avg:142.41ms
step:108/1480 train_time:13957ms step_avg:142.42ms
step:109/1480 train_time:14101ms step_avg:142.43ms
step:110/1480 train_time:14244ms step_avg:142.44ms
step:111/1480 train_time:14387ms step_avg:142.44ms
step:112/1480 train_time:14534ms step_avg:142.49ms
step:113/1480 train_time:14681ms step_avg:142.54ms
step:114/1480 train_time:14827ms step_avg:142.57ms
step:115/1480 train_time:14975ms step_avg:142.62ms
step:116/1480 train_time:15122ms step_avg:142.66ms
step:117/1480 train_time:15269ms step_avg:142.70ms
step:118/1480 train_time:15416ms step_avg:142.74ms
step:119/1480 train_time:15564ms step_avg:142.78ms
step:120/1480 train_time:15709ms step_avg:142.81ms
step:121/1480 train_time:15856ms step_avg:142.85ms
step:122/1480 train_time:16003ms step_avg:142.89ms
step:123/1480 train_time:16149ms step_avg:142.91ms
step:124/1480 train_time:16295ms step_avg:142.94ms
step:125/1480 train_time:16442ms step_avg:142.97ms
step:125/1480 val_loss:4.4112 train_time:16498ms step_avg:143.46ms
step:126/1480 train_time:16595ms step_avg:143.06ms
step:127/1480 train_time:16745ms step_avg:143.12ms
step:128/1480 train_time:16890ms step_avg:143.14ms
step:129/1480 train_time:17036ms step_avg:143.16ms
step:130/1480 train_time:17182ms step_avg:143.18ms
step:131/1480 train_time:17328ms step_avg:143.21ms
step:132/1480 train_time:17472ms step_avg:143.21ms
step:133/1480 train_time:17622ms step_avg:143.27ms
step:134/1480 train_time:17769ms step_avg:143.30ms
step:135/1480 train_time:17916ms step_avg:143.33ms
step:136/1480 train_time:18063ms step_avg:143.36ms
step:137/1480 train_time:18209ms step_avg:143.38ms
step:138/1480 train_time:18356ms step_avg:143.41ms
step:139/1480 train_time:18504ms step_avg:143.44ms
step:140/1480 train_time:18650ms step_avg:143.46ms
step:141/1480 train_time:18797ms step_avg:143.49ms
step:142/1480 train_time:18945ms step_avg:143.52ms
step:143/1480 train_time:19091ms step_avg:143.54ms
step:144/1480 train_time:19237ms step_avg:143.56ms
step:145/1480 train_time:19384ms step_avg:143.59ms
step:146/1480 train_time:19530ms step_avg:143.60ms
step:147/1480 train_time:19677ms step_avg:143.63ms
step:148/1480 train_time:19824ms step_avg:143.65ms
step:149/1480 train_time:19970ms step_avg:143.67ms
step:150/1480 train_time:20118ms step_avg:143.70ms
step:151/1480 train_time:20265ms step_avg:143.72ms
step:152/1480 train_time:20410ms step_avg:143.73ms
step:153/1480 train_time:20557ms step_avg:143.76ms
step:154/1480 train_time:20705ms step_avg:143.79ms
step:155/1480 train_time:20851ms step_avg:143.80ms
step:156/1480 train_time:20997ms step_avg:143.82ms
step:157/1480 train_time:21145ms step_avg:143.84ms
step:158/1480 train_time:21291ms step_avg:143.86ms
step:159/1480 train_time:21439ms step_avg:143.88ms
step:160/1480 train_time:21586ms step_avg:143.91ms
step:161/1480 train_time:21731ms step_avg:143.92ms
step:162/1480 train_time:21880ms step_avg:143.95ms
step:163/1480 train_time:22027ms step_avg:143.97ms
step:164/1480 train_time:22172ms step_avg:143.98ms
step:165/1480 train_time:22319ms step_avg:144.00ms
step:166/1480 train_time:22466ms step_avg:144.01ms
step:167/1480 train_time:22612ms step_avg:144.03ms
step:168/1480 train_time:22760ms step_avg:144.05ms
step:169/1480 train_time:22907ms step_avg:144.07ms
step:170/1480 train_time:23053ms step_avg:144.08ms
step:171/1480 train_time:23201ms step_avg:144.10ms
step:172/1480 train_time:23347ms step_avg:144.12ms
step:173/1480 train_time:23492ms step_avg:144.12ms
step:174/1480 train_time:23638ms step_avg:144.13ms
step:175/1480 train_time:23785ms step_avg:144.15ms
step:176/1480 train_time:23930ms step_avg:144.15ms
step:177/1480 train_time:24078ms step_avg:144.18ms
step:178/1480 train_time:24227ms step_avg:144.21ms
step:179/1480 train_time:24372ms step_avg:144.21ms
step:180/1480 train_time:24519ms step_avg:144.23ms
step:181/1480 train_time:24666ms step_avg:144.24ms
step:182/1480 train_time:24812ms step_avg:144.26ms
step:183/1480 train_time:24960ms step_avg:144.28ms
step:184/1480 train_time:25108ms step_avg:144.30ms
step:185/1480 train_time:25253ms step_avg:144.30ms
step:186/1480 train_time:25400ms step_avg:144.32ms
step:187/1480 train_time:25546ms step_avg:144.33ms
step:188/1480 train_time:25692ms step_avg:144.34ms
step:189/1480 train_time:25839ms step_avg:144.35ms
step:190/1480 train_time:25986ms step_avg:144.37ms
step:191/1480 train_time:26131ms step_avg:144.37ms
step:192/1480 train_time:26279ms step_avg:144.39ms
step:193/1480 train_time:26426ms step_avg:144.41ms
step:194/1480 train_time:26572ms step_avg:144.41ms
step:195/1480 train_time:26719ms step_avg:144.43ms
step:196/1480 train_time:26867ms step_avg:144.44ms
step:197/1480 train_time:27012ms step_avg:144.45ms
step:198/1480 train_time:27160ms step_avg:144.47ms
step:199/1480 train_time:27307ms step_avg:144.48ms
step:200/1480 train_time:27452ms step_avg:144.49ms
step:201/1480 train_time:27599ms step_avg:144.50ms
step:202/1480 train_time:27746ms step_avg:144.51ms
step:203/1480 train_time:27891ms step_avg:144.51ms
step:204/1480 train_time:28037ms step_avg:144.52ms
step:205/1480 train_time:28185ms step_avg:144.54ms
step:206/1480 train_time:28331ms step_avg:144.55ms
step:207/1480 train_time:28478ms step_avg:144.56ms
step:208/1480 train_time:28626ms step_avg:144.58ms
step:209/1480 train_time:28773ms step_avg:144.59ms
step:210/1480 train_time:28920ms step_avg:144.60ms
step:211/1480 train_time:29066ms step_avg:144.61ms
step:212/1480 train_time:29212ms step_avg:144.61ms
step:213/1480 train_time:29360ms step_avg:144.63ms
step:214/1480 train_time:29507ms step_avg:144.64ms
step:215/1480 train_time:29653ms step_avg:144.65ms
step:216/1480 train_time:29801ms step_avg:144.67ms
step:217/1480 train_time:29948ms step_avg:144.68ms
step:218/1480 train_time:30093ms step_avg:144.68ms
step:219/1480 train_time:30239ms step_avg:144.69ms
step:220/1480 train_time:30387ms step_avg:144.70ms
step:221/1480 train_time:30534ms step_avg:144.71ms
step:222/1480 train_time:30685ms step_avg:144.74ms
step:223/1480 train_time:30834ms step_avg:144.76ms
step:224/1480 train_time:30985ms step_avg:144.79ms
step:225/1480 train_time:31135ms step_avg:144.81ms
step:226/1480 train_time:31286ms step_avg:144.84ms
step:227/1480 train_time:31435ms step_avg:144.86ms
step:228/1480 train_time:31586ms step_avg:144.89ms
step:229/1480 train_time:31735ms step_avg:144.91ms
step:230/1480 train_time:31887ms step_avg:144.94ms
step:231/1480 train_time:32036ms step_avg:144.96ms
step:232/1480 train_time:32187ms step_avg:144.99ms
step:233/1480 train_time:32336ms step_avg:145.00ms
step:234/1480 train_time:32488ms step_avg:145.03ms
step:235/1480 train_time:32639ms step_avg:145.06ms
step:236/1480 train_time:32789ms step_avg:145.08ms
step:237/1480 train_time:32939ms step_avg:145.10ms
step:238/1480 train_time:33090ms step_avg:145.13ms
step:239/1480 train_time:33239ms step_avg:145.15ms
step:240/1480 train_time:33390ms step_avg:145.17ms
step:241/1480 train_time:33539ms step_avg:145.19ms
step:242/1480 train_time:33688ms step_avg:145.21ms
step:243/1480 train_time:33837ms step_avg:145.22ms
step:244/1480 train_time:33989ms step_avg:145.25ms
step:245/1480 train_time:34138ms step_avg:145.27ms
step:246/1480 train_time:34290ms step_avg:145.30ms
step:247/1480 train_time:34440ms step_avg:145.32ms
step:248/1480 train_time:34591ms step_avg:145.34ms
step:249/1480 train_time:34742ms step_avg:145.37ms
step:250/1480 train_time:34894ms step_avg:145.39ms
step:250/1480 val_loss:4.0001 train_time:34953ms step_avg:145.64ms
step:251/1480 train_time:35050ms step_avg:145.44ms
step:252/1480 train_time:35203ms step_avg:145.47ms
step:253/1480 train_time:35352ms step_avg:145.48ms
step:254/1480 train_time:35501ms step_avg:145.50ms
step:255/1480 train_time:35650ms step_avg:145.51ms
step:256/1480 train_time:35800ms step_avg:145.53ms
step:257/1480 train_time:35950ms step_avg:145.55ms
step:258/1480 train_time:36104ms step_avg:145.58ms
step:259/1480 train_time:36254ms step_avg:145.60ms
step:260/1480 train_time:36405ms step_avg:145.62ms
step:261/1480 train_time:36555ms step_avg:145.64ms
step:262/1480 train_time:36705ms step_avg:145.66ms
step:263/1480 train_time:36854ms step_avg:145.67ms
step:264/1480 train_time:37006ms step_avg:145.69ms
step:265/1480 train_time:37156ms step_avg:145.71ms
step:266/1480 train_time:37307ms step_avg:145.73ms
step:267/1480 train_time:37456ms step_avg:145.74ms
step:268/1480 train_time:37608ms step_avg:145.77ms
step:269/1480 train_time:37757ms step_avg:145.78ms
step:270/1480 train_time:37909ms step_avg:145.80ms
step:271/1480 train_time:38058ms step_avg:145.81ms
step:272/1480 train_time:38209ms step_avg:145.83ms
step:273/1480 train_time:38358ms step_avg:145.85ms
step:274/1480 train_time:38509ms step_avg:145.87ms
step:275/1480 train_time:38659ms step_avg:145.88ms
step:276/1480 train_time:38809ms step_avg:145.90ms
step:277/1480 train_time:38958ms step_avg:145.91ms
step:278/1480 train_time:39111ms step_avg:145.93ms
step:279/1480 train_time:39261ms step_avg:145.95ms
step:280/1480 train_time:39413ms step_avg:145.97ms
step:281/1480 train_time:39564ms step_avg:145.99ms
step:282/1480 train_time:39715ms step_avg:146.01ms
step:283/1480 train_time:39865ms step_avg:146.03ms
step:284/1480 train_time:40015ms step_avg:146.04ms
step:285/1480 train_time:40166ms step_avg:146.06ms
step:286/1480 train_time:40316ms step_avg:146.07ms
step:287/1480 train_time:40467ms step_avg:146.09ms
step:288/1480 train_time:40617ms step_avg:146.10ms
step:289/1480 train_time:40768ms step_avg:146.12ms
step:290/1480 train_time:40918ms step_avg:146.14ms
step:291/1480 train_time:41068ms step_avg:146.15ms
step:292/1480 train_time:41220ms step_avg:146.17ms
step:293/1480 train_time:41368ms step_avg:146.18ms
step:294/1480 train_time:41519ms step_avg:146.19ms
step:295/1480 train_time:41669ms step_avg:146.21ms
step:296/1480 train_time:41820ms step_avg:146.22ms
step:297/1480 train_time:41970ms step_avg:146.24ms
step:298/1480 train_time:42120ms step_avg:146.25ms
step:299/1480 train_time:42270ms step_avg:146.26ms
step:300/1480 train_time:42420ms step_avg:146.28ms
step:301/1480 train_time:42570ms step_avg:146.29ms
step:302/1480 train_time:42719ms step_avg:146.30ms
step:303/1480 train_time:42870ms step_avg:146.31ms
step:304/1480 train_time:43018ms step_avg:146.32ms
step:305/1480 train_time:43169ms step_avg:146.34ms
step:306/1480 train_time:43320ms step_avg:146.35ms
step:307/1480 train_time:43470ms step_avg:146.37ms
step:308/1480 train_time:43621ms step_avg:146.38ms
step:309/1480 train_time:43772ms step_avg:146.39ms
step:310/1480 train_time:43924ms step_avg:146.41ms
step:311/1480 train_time:44074ms step_avg:146.42ms
step:312/1480 train_time:44225ms step_avg:146.44ms
step:313/1480 train_time:44375ms step_avg:146.45ms
step:314/1480 train_time:44525ms step_avg:146.47ms
step:315/1480 train_time:44674ms step_avg:146.47ms
step:316/1480 train_time:44825ms step_avg:146.49ms
step:317/1480 train_time:44975ms step_avg:146.50ms
step:318/1480 train_time:45126ms step_avg:146.51ms
step:319/1480 train_time:45275ms step_avg:146.52ms
step:320/1480 train_time:45426ms step_avg:146.54ms
step:321/1480 train_time:45576ms step_avg:146.55ms
step:322/1480 train_time:45727ms step_avg:146.56ms
step:323/1480 train_time:45876ms step_avg:146.57ms
step:324/1480 train_time:46027ms step_avg:146.58ms
step:325/1480 train_time:46177ms step_avg:146.59ms
step:326/1480 train_time:46328ms step_avg:146.61ms
step:327/1480 train_time:46479ms step_avg:146.62ms
step:328/1480 train_time:46630ms step_avg:146.64ms
step:329/1480 train_time:46780ms step_avg:146.65ms
step:330/1480 train_time:46933ms step_avg:146.66ms
step:331/1480 train_time:47086ms step_avg:146.69ms
step:332/1480 train_time:47240ms step_avg:146.71ms
step:333/1480 train_time:47393ms step_avg:146.73ms
step:334/1480 train_time:47547ms step_avg:146.75ms
step:335/1480 train_time:47700ms step_avg:146.77ms
step:336/1480 train_time:47854ms step_avg:146.79ms
step:337/1480 train_time:48009ms step_avg:146.82ms
step:338/1480 train_time:48163ms step_avg:146.84ms
step:339/1480 train_time:48320ms step_avg:146.87ms
step:340/1480 train_time:48471ms step_avg:146.88ms
step:341/1480 train_time:48626ms step_avg:146.91ms
step:342/1480 train_time:48779ms step_avg:146.92ms
step:343/1480 train_time:48933ms step_avg:146.95ms
step:344/1480 train_time:49086ms step_avg:146.97ms
step:345/1480 train_time:49241ms step_avg:146.99ms
step:346/1480 train_time:49395ms step_avg:147.01ms
step:347/1480 train_time:49549ms step_avg:147.03ms
step:348/1480 train_time:49702ms step_avg:147.05ms
step:349/1480 train_time:49856ms step_avg:147.07ms
step:350/1480 train_time:50010ms step_avg:147.09ms
step:351/1480 train_time:50163ms step_avg:147.10ms
step:352/1480 train_time:50317ms step_avg:147.13ms
step:353/1480 train_time:50471ms step_avg:147.15ms
step:354/1480 train_time:50626ms step_avg:147.17ms
step:355/1480 train_time:50780ms step_avg:147.19ms
step:356/1480 train_time:50933ms step_avg:147.21ms
step:357/1480 train_time:51086ms step_avg:147.22ms
step:358/1480 train_time:51239ms step_avg:147.24ms
step:359/1480 train_time:51394ms step_avg:147.26ms
step:360/1480 train_time:51549ms step_avg:147.28ms
step:361/1480 train_time:51704ms step_avg:147.30ms
step:362/1480 train_time:51858ms step_avg:147.32ms
step:363/1480 train_time:52010ms step_avg:147.34ms
step:364/1480 train_time:52163ms step_avg:147.35ms
step:365/1480 train_time:52316ms step_avg:147.37ms
step:366/1480 train_time:52469ms step_avg:147.38ms
step:367/1480 train_time:52621ms step_avg:147.40ms
step:368/1480 train_time:52775ms step_avg:147.42ms
step:369/1480 train_time:52930ms step_avg:147.44ms
step:370/1480 train_time:53084ms step_avg:147.45ms
step:371/1480 train_time:53239ms step_avg:147.48ms
step:372/1480 train_time:53392ms step_avg:147.49ms
step:373/1480 train_time:53546ms step_avg:147.51ms
step:374/1480 train_time:53698ms step_avg:147.52ms
step:375/1480 train_time:53851ms step_avg:147.54ms
step:375/1480 val_loss:3.8110 train_time:53910ms step_avg:147.70ms
step:376/1480 train_time:54010ms step_avg:147.57ms
step:377/1480 train_time:54164ms step_avg:147.59ms
step:378/1480 train_time:54316ms step_avg:147.60ms
step:379/1480 train_time:54469ms step_avg:147.61ms
step:380/1480 train_time:54622ms step_avg:147.63ms
step:381/1480 train_time:54774ms step_avg:147.64ms
step:382/1480 train_time:54929ms step_avg:147.66ms
step:383/1480 train_time:55085ms step_avg:147.68ms
step:384/1480 train_time:55239ms step_avg:147.70ms
step:385/1480 train_time:55392ms step_avg:147.71ms
step:386/1480 train_time:55547ms step_avg:147.73ms
step:387/1480 train_time:55700ms step_avg:147.74ms
step:388/1480 train_time:55852ms step_avg:147.76ms
step:389/1480 train_time:56006ms step_avg:147.77ms
step:390/1480 train_time:56160ms step_avg:147.79ms
step:391/1480 train_time:56314ms step_avg:147.81ms
step:392/1480 train_time:56467ms step_avg:147.82ms
step:393/1480 train_time:56621ms step_avg:147.84ms
step:394/1480 train_time:56774ms step_avg:147.85ms
step:395/1480 train_time:56927ms step_avg:147.86ms
step:396/1480 train_time:57081ms step_avg:147.88ms
step:397/1480 train_time:57235ms step_avg:147.89ms
step:398/1480 train_time:57389ms step_avg:147.91ms
step:399/1480 train_time:57543ms step_avg:147.93ms
step:400/1480 train_time:57697ms step_avg:147.94ms
step:401/1480 train_time:57851ms step_avg:147.96ms
step:402/1480 train_time:58004ms step_avg:147.97ms
step:403/1480 train_time:58157ms step_avg:147.98ms
step:404/1480 train_time:58312ms step_avg:148.00ms
step:405/1480 train_time:58467ms step_avg:148.02ms
step:406/1480 train_time:58621ms step_avg:148.03ms
step:407/1480 train_time:58776ms step_avg:148.05ms
step:408/1480 train_time:58928ms step_avg:148.06ms
step:409/1480 train_time:59083ms step_avg:148.08ms
step:410/1480 train_time:59236ms step_avg:148.09ms
step:411/1480 train_time:59390ms step_avg:148.11ms
step:412/1480 train_time:59546ms step_avg:148.13ms
step:413/1480 train_time:59700ms step_avg:148.14ms
step:414/1480 train_time:59854ms step_avg:148.15ms
step:415/1480 train_time:60009ms step_avg:148.17ms
step:416/1480 train_time:60162ms step_avg:148.18ms
step:417/1480 train_time:60315ms step_avg:148.20ms
step:418/1480 train_time:60470ms step_avg:148.21ms
step:419/1480 train_time:60622ms step_avg:148.22ms
step:420/1480 train_time:60775ms step_avg:148.23ms
step:421/1480 train_time:60929ms step_avg:148.24ms
step:422/1480 train_time:61082ms step_avg:148.26ms
step:423/1480 train_time:61235ms step_avg:148.27ms
step:424/1480 train_time:61388ms step_avg:148.28ms
step:425/1480 train_time:61543ms step_avg:148.30ms
step:426/1480 train_time:61697ms step_avg:148.31ms
step:427/1480 train_time:61851ms step_avg:148.32ms
step:428/1480 train_time:62005ms step_avg:148.34ms
step:429/1480 train_time:62158ms step_avg:148.35ms
step:430/1480 train_time:62312ms step_avg:148.36ms
step:431/1480 train_time:62465ms step_avg:148.37ms
step:432/1480 train_time:62619ms step_avg:148.39ms
step:433/1480 train_time:62772ms step_avg:148.40ms
step:434/1480 train_time:62927ms step_avg:148.41ms
step:435/1480 train_time:63081ms step_avg:148.43ms
step:436/1480 train_time:63234ms step_avg:148.44ms
step:437/1480 train_time:63387ms step_avg:148.45ms
step:438/1480 train_time:63541ms step_avg:148.46ms
step:439/1480 train_time:63695ms step_avg:148.47ms
step:440/1480 train_time:63850ms step_avg:148.49ms
step:441/1480 train_time:64008ms step_avg:148.51ms
step:442/1480 train_time:64165ms step_avg:148.53ms
step:443/1480 train_time:64322ms step_avg:148.55ms
step:444/1480 train_time:64478ms step_avg:148.57ms
step:445/1480 train_time:64634ms step_avg:148.58ms
step:446/1480 train_time:64789ms step_avg:148.60ms
step:447/1480 train_time:64945ms step_avg:148.62ms
step:448/1480 train_time:65102ms step_avg:148.64ms
step:449/1480 train_time:65261ms step_avg:148.66ms
step:450/1480 train_time:65417ms step_avg:148.68ms
step:451/1480 train_time:65573ms step_avg:148.69ms
step:452/1480 train_time:65729ms step_avg:148.71ms
step:453/1480 train_time:65885ms step_avg:148.72ms
step:454/1480 train_time:66042ms step_avg:148.74ms
step:455/1480 train_time:66199ms step_avg:148.76ms
step:456/1480 train_time:66355ms step_avg:148.78ms
step:457/1480 train_time:66512ms step_avg:148.80ms
step:458/1480 train_time:66668ms step_avg:148.81ms
step:459/1480 train_time:66827ms step_avg:148.83ms
step:460/1480 train_time:66984ms step_avg:148.85ms
step:461/1480 train_time:67144ms step_avg:148.88ms
step:462/1480 train_time:67302ms step_avg:148.90ms
step:463/1480 train_time:67459ms step_avg:148.92ms
step:464/1480 train_time:67615ms step_avg:148.93ms
step:465/1480 train_time:67771ms step_avg:148.95ms
step:466/1480 train_time:67927ms step_avg:148.96ms
step:467/1480 train_time:68083ms step_avg:148.98ms
step:468/1480 train_time:68240ms step_avg:149.00ms
step:469/1480 train_time:68396ms step_avg:149.01ms
step:470/1480 train_time:68553ms step_avg:149.03ms
step:471/1480 train_time:68709ms step_avg:149.04ms
step:472/1480 train_time:68866ms step_avg:149.06ms
step:473/1480 train_time:69023ms step_avg:149.08ms
step:474/1480 train_time:69180ms step_avg:149.09ms
step:475/1480 train_time:69335ms step_avg:149.11ms
step:476/1480 train_time:69492ms step_avg:149.13ms
step:477/1480 train_time:69650ms step_avg:149.14ms
step:478/1480 train_time:69807ms step_avg:149.16ms
step:479/1480 train_time:69964ms step_avg:149.18ms
step:480/1480 train_time:70123ms step_avg:149.20ms
step:481/1480 train_time:70281ms step_avg:149.22ms
step:482/1480 train_time:70436ms step_avg:149.23ms
step:483/1480 train_time:70592ms step_avg:149.24ms
step:484/1480 train_time:70750ms step_avg:149.26ms
step:485/1480 train_time:70907ms step_avg:149.28ms
step:486/1480 train_time:71065ms step_avg:149.30ms
step:487/1480 train_time:71224ms step_avg:149.32ms
step:488/1480 train_time:71382ms step_avg:149.33ms
step:489/1480 train_time:71538ms step_avg:149.35ms
step:490/1480 train_time:71695ms step_avg:149.36ms
step:491/1480 train_time:71850ms step_avg:149.38ms
step:492/1480 train_time:72007ms step_avg:149.39ms
step:493/1480 train_time:72163ms step_avg:149.41ms
step:494/1480 train_time:72321ms step_avg:149.42ms
step:495/1480 train_time:72477ms step_avg:149.44ms
step:496/1480 train_time:72635ms step_avg:149.45ms
step:497/1480 train_time:72790ms step_avg:149.47ms
step:498/1480 train_time:72948ms step_avg:149.48ms
step:499/1480 train_time:73106ms step_avg:149.50ms
step:500/1480 train_time:73264ms step_avg:149.52ms
step:500/1480 val_loss:3.6905 train_time:73326ms step_avg:149.64ms
step:501/1480 train_time:73423ms step_avg:149.54ms
step:502/1480 train_time:73579ms step_avg:149.55ms
step:503/1480 train_time:73737ms step_avg:149.57ms
step:504/1480 train_time:73893ms step_avg:149.58ms
step:505/1480 train_time:74048ms step_avg:149.59ms
step:506/1480 train_time:74204ms step_avg:149.61ms
step:507/1480 train_time:74360ms step_avg:149.62ms
step:508/1480 train_time:74519ms step_avg:149.64ms
step:509/1480 train_time:74675ms step_avg:149.65ms
step:510/1480 train_time:74834ms step_avg:149.67ms
step:511/1480 train_time:74990ms step_avg:149.68ms
step:512/1480 train_time:75148ms step_avg:149.70ms
step:513/1480 train_time:75302ms step_avg:149.71ms
step:514/1480 train_time:75459ms step_avg:149.72ms
step:515/1480 train_time:75616ms step_avg:149.74ms
step:516/1480 train_time:75774ms step_avg:149.75ms
step:517/1480 train_time:75933ms step_avg:149.77ms
step:518/1480 train_time:76091ms step_avg:149.79ms
step:519/1480 train_time:76249ms step_avg:149.80ms
step:520/1480 train_time:76406ms step_avg:149.82ms
step:521/1480 train_time:76562ms step_avg:149.83ms
step:522/1480 train_time:76719ms step_avg:149.84ms
step:523/1480 train_time:76876ms step_avg:149.86ms
step:524/1480 train_time:77034ms step_avg:149.87ms
step:525/1480 train_time:77191ms step_avg:149.89ms
step:526/1480 train_time:77350ms step_avg:149.90ms
step:527/1480 train_time:77506ms step_avg:149.91ms
step:528/1480 train_time:77661ms step_avg:149.92ms
step:529/1480 train_time:77817ms step_avg:149.94ms
step:530/1480 train_time:77975ms step_avg:149.95ms
step:531/1480 train_time:78132ms step_avg:149.96ms
step:532/1480 train_time:78288ms step_avg:149.98ms
step:533/1480 train_time:78444ms step_avg:149.99ms
step:534/1480 train_time:78600ms step_avg:150.00ms
step:535/1480 train_time:78756ms step_avg:150.01ms
step:536/1480 train_time:78913ms step_avg:150.03ms
step:537/1480 train_time:79071ms step_avg:150.04ms
step:538/1480 train_time:79229ms step_avg:150.05ms
step:539/1480 train_time:79387ms step_avg:150.07ms
step:540/1480 train_time:79543ms step_avg:150.08ms
step:541/1480 train_time:79700ms step_avg:150.09ms
step:542/1480 train_time:79856ms step_avg:150.11ms
step:543/1480 train_time:80013ms step_avg:150.12ms
step:544/1480 train_time:80170ms step_avg:150.13ms
step:545/1480 train_time:80327ms step_avg:150.14ms
step:546/1480 train_time:80483ms step_avg:150.15ms
step:547/1480 train_time:80640ms step_avg:150.17ms
step:548/1480 train_time:80798ms step_avg:150.18ms
step:549/1480 train_time:80954ms step_avg:150.19ms
step:550/1480 train_time:81114ms step_avg:150.21ms
step:551/1480 train_time:81273ms step_avg:150.23ms
step:552/1480 train_time:81433ms step_avg:150.25ms
step:553/1480 train_time:81594ms step_avg:150.26ms
step:554/1480 train_time:81754ms step_avg:150.28ms
step:555/1480 train_time:81915ms step_avg:150.30ms
step:556/1480 train_time:82074ms step_avg:150.32ms
step:557/1480 train_time:82235ms step_avg:150.34ms
step:558/1480 train_time:82395ms step_avg:150.36ms
step:559/1480 train_time:82555ms step_avg:150.37ms
step:560/1480 train_time:82715ms step_avg:150.39ms
step:561/1480 train_time:82875ms step_avg:150.41ms
step:562/1480 train_time:83035ms step_avg:150.43ms
step:563/1480 train_time:83194ms step_avg:150.44ms
step:564/1480 train_time:83353ms step_avg:150.46ms
step:565/1480 train_time:83513ms step_avg:150.47ms
step:566/1480 train_time:83673ms step_avg:150.49ms
step:567/1480 train_time:83833ms step_avg:150.51ms
step:568/1480 train_time:83992ms step_avg:150.52ms
step:569/1480 train_time:84150ms step_avg:150.54ms
step:570/1480 train_time:84309ms step_avg:150.55ms
step:571/1480 train_time:84468ms step_avg:150.57ms
step:572/1480 train_time:84627ms step_avg:150.58ms
step:573/1480 train_time:84789ms step_avg:150.60ms
step:574/1480 train_time:84950ms step_avg:150.62ms
step:575/1480 train_time:85113ms step_avg:150.64ms
step:576/1480 train_time:85272ms step_avg:150.66ms
step:577/1480 train_time:85433ms step_avg:150.68ms
step:578/1480 train_time:85592ms step_avg:150.69ms
step:579/1480 train_time:85752ms step_avg:150.71ms
step:580/1480 train_time:85911ms step_avg:150.72ms
step:581/1480 train_time:86071ms step_avg:150.74ms
step:582/1480 train_time:86232ms step_avg:150.76ms
step:583/1480 train_time:86392ms step_avg:150.77ms
step:584/1480 train_time:86551ms step_avg:150.79ms
step:585/1480 train_time:86710ms step_avg:150.80ms
step:586/1480 train_time:86871ms step_avg:150.82ms
step:587/1480 train_time:87031ms step_avg:150.83ms
step:588/1480 train_time:87191ms step_avg:150.85ms
step:589/1480 train_time:87350ms step_avg:150.86ms
step:590/1480 train_time:87511ms step_avg:150.88ms
step:591/1480 train_time:87670ms step_avg:150.89ms
step:592/1480 train_time:87828ms step_avg:150.91ms
step:593/1480 train_time:87989ms step_avg:150.92ms
step:594/1480 train_time:88149ms step_avg:150.94ms
step:595/1480 train_time:88310ms step_avg:150.96ms
step:596/1480 train_time:88473ms step_avg:150.98ms
step:597/1480 train_time:88635ms step_avg:151.00ms
step:598/1480 train_time:88793ms step_avg:151.01ms
step:599/1480 train_time:88951ms step_avg:151.02ms
step:600/1480 train_time:89111ms step_avg:151.04ms
step:601/1480 train_time:89271ms step_avg:151.05ms
step:602/1480 train_time:89431ms step_avg:151.07ms
step:603/1480 train_time:89592ms step_avg:151.08ms
step:604/1480 train_time:89752ms step_avg:151.10ms
step:605/1480 train_time:89912ms step_avg:151.11ms
step:606/1480 train_time:90074ms step_avg:151.13ms
step:607/1480 train_time:90237ms step_avg:151.15ms
step:608/1480 train_time:90397ms step_avg:151.17ms
step:609/1480 train_time:90556ms step_avg:151.18ms
step:610/1480 train_time:90715ms step_avg:151.19ms
step:611/1480 train_time:90875ms step_avg:151.21ms
step:612/1480 train_time:91037ms step_avg:151.22ms
step:613/1480 train_time:91198ms step_avg:151.24ms
step:614/1480 train_time:91358ms step_avg:151.25ms
step:615/1480 train_time:91517ms step_avg:151.27ms
step:616/1480 train_time:91675ms step_avg:151.28ms
step:617/1480 train_time:91834ms step_avg:151.29ms
step:618/1480 train_time:91993ms step_avg:151.30ms
step:619/1480 train_time:92153ms step_avg:151.32ms
step:620/1480 train_time:92313ms step_avg:151.33ms
step:621/1480 train_time:92473ms step_avg:151.35ms
step:622/1480 train_time:92634ms step_avg:151.36ms
step:623/1480 train_time:92795ms step_avg:151.38ms
step:624/1480 train_time:92955ms step_avg:151.39ms
step:625/1480 train_time:93115ms step_avg:151.41ms
step:625/1480 val_loss:3.6069 train_time:93178ms step_avg:151.51ms
step:626/1480 train_time:93276ms step_avg:151.42ms
step:627/1480 train_time:93434ms step_avg:151.43ms
step:628/1480 train_time:93591ms step_avg:151.44ms
step:629/1480 train_time:93748ms step_avg:151.45ms
step:630/1480 train_time:93907ms step_avg:151.46ms
step:631/1480 train_time:94066ms step_avg:151.48ms
step:632/1480 train_time:94225ms step_avg:151.49ms
step:633/1480 train_time:94385ms step_avg:151.50ms
step:634/1480 train_time:94545ms step_avg:151.52ms
step:635/1480 train_time:94707ms step_avg:151.53ms
step:636/1480 train_time:94867ms step_avg:151.54ms
step:637/1480 train_time:95027ms step_avg:151.56ms
step:638/1480 train_time:95186ms step_avg:151.57ms
step:639/1480 train_time:95345ms step_avg:151.58ms
step:640/1480 train_time:95506ms step_avg:151.60ms
step:641/1480 train_time:95665ms step_avg:151.61ms
step:642/1480 train_time:95824ms step_avg:151.62ms
step:643/1480 train_time:95986ms step_avg:151.64ms
step:644/1480 train_time:96145ms step_avg:151.65ms
step:645/1480 train_time:96303ms step_avg:151.66ms
step:646/1480 train_time:96461ms step_avg:151.67ms
step:647/1480 train_time:96618ms step_avg:151.68ms
step:648/1480 train_time:96780ms step_avg:151.69ms
step:649/1480 train_time:96938ms step_avg:151.70ms
step:650/1480 train_time:97099ms step_avg:151.72ms
step:651/1480 train_time:97259ms step_avg:151.73ms
step:652/1480 train_time:97418ms step_avg:151.74ms
step:653/1480 train_time:97577ms step_avg:151.75ms
step:654/1480 train_time:97737ms step_avg:151.77ms
step:655/1480 train_time:97896ms step_avg:151.78ms
step:656/1480 train_time:98055ms step_avg:151.79ms
step:657/1480 train_time:98214ms step_avg:151.80ms
step:658/1480 train_time:98373ms step_avg:151.81ms
step:659/1480 train_time:98533ms step_avg:151.82ms
step:660/1480 train_time:98694ms step_avg:151.84ms
step:661/1480 train_time:98856ms step_avg:151.85ms
step:662/1480 train_time:99016ms step_avg:151.86ms
step:663/1480 train_time:99176ms step_avg:151.88ms
step:664/1480 train_time:99337ms step_avg:151.89ms
step:665/1480 train_time:99499ms step_avg:151.91ms
step:666/1480 train_time:99659ms step_avg:151.92ms
step:667/1480 train_time:99822ms step_avg:151.94ms
step:668/1480 train_time:99988ms step_avg:151.96ms
step:669/1480 train_time:100150ms step_avg:151.97ms
step:670/1480 train_time:100310ms step_avg:151.99ms
step:671/1480 train_time:100471ms step_avg:152.00ms
step:672/1480 train_time:100632ms step_avg:152.01ms
step:673/1480 train_time:100794ms step_avg:152.03ms
step:674/1480 train_time:100955ms step_avg:152.04ms
step:675/1480 train_time:101116ms step_avg:152.05ms
step:676/1480 train_time:101279ms step_avg:152.07ms
step:677/1480 train_time:101440ms step_avg:152.08ms
step:678/1480 train_time:101602ms step_avg:152.10ms
step:679/1480 train_time:101764ms step_avg:152.11ms
step:680/1480 train_time:101928ms step_avg:152.13ms
step:681/1480 train_time:102088ms step_avg:152.14ms
step:682/1480 train_time:102250ms step_avg:152.16ms
step:683/1480 train_time:102412ms step_avg:152.17ms
step:684/1480 train_time:102572ms step_avg:152.18ms
step:685/1480 train_time:102734ms step_avg:152.20ms
step:686/1480 train_time:102896ms step_avg:152.21ms
step:687/1480 train_time:103056ms step_avg:152.22ms
step:688/1480 train_time:103218ms step_avg:152.24ms
step:689/1480 train_time:103381ms step_avg:152.25ms
step:690/1480 train_time:103544ms step_avg:152.27ms
step:691/1480 train_time:103707ms step_avg:152.29ms
step:692/1480 train_time:103869ms step_avg:152.30ms
step:693/1480 train_time:104031ms step_avg:152.31ms
step:694/1480 train_time:104192ms step_avg:152.33ms
step:695/1480 train_time:104351ms step_avg:152.34ms
step:696/1480 train_time:104512ms step_avg:152.35ms
step:697/1480 train_time:104675ms step_avg:152.36ms
step:698/1480 train_time:104835ms step_avg:152.38ms
step:699/1480 train_time:104997ms step_avg:152.39ms
step:700/1480 train_time:105158ms step_avg:152.40ms
step:701/1480 train_time:105318ms step_avg:152.41ms
step:702/1480 train_time:105479ms step_avg:152.43ms
step:703/1480 train_time:105640ms step_avg:152.44ms
step:704/1480 train_time:105800ms step_avg:152.45ms
step:705/1480 train_time:105964ms step_avg:152.47ms
step:706/1480 train_time:106127ms step_avg:152.48ms
step:707/1480 train_time:106289ms step_avg:152.49ms
step:708/1480 train_time:106449ms step_avg:152.51ms
step:709/1480 train_time:106610ms step_avg:152.52ms
step:710/1480 train_time:106770ms step_avg:152.53ms
step:711/1480 train_time:106932ms step_avg:152.54ms
step:712/1480 train_time:107097ms step_avg:152.56ms
step:713/1480 train_time:107262ms step_avg:152.58ms
step:714/1480 train_time:107424ms step_avg:152.59ms
step:715/1480 train_time:107585ms step_avg:152.60ms
step:716/1480 train_time:107746ms step_avg:152.61ms
step:717/1480 train_time:107910ms step_avg:152.63ms
step:718/1480 train_time:108070ms step_avg:152.64ms
step:719/1480 train_time:108229ms step_avg:152.65ms
step:720/1480 train_time:108392ms step_avg:152.66ms
step:721/1480 train_time:108553ms step_avg:152.68ms
step:722/1480 train_time:108715ms step_avg:152.69ms
step:723/1480 train_time:108875ms step_avg:152.70ms
step:724/1480 train_time:109037ms step_avg:152.71ms
step:725/1480 train_time:109198ms step_avg:152.72ms
step:726/1480 train_time:109362ms step_avg:152.74ms
step:727/1480 train_time:109525ms step_avg:152.76ms
step:728/1480 train_time:109688ms step_avg:152.77ms
step:729/1480 train_time:109849ms step_avg:152.78ms
step:730/1480 train_time:110013ms step_avg:152.80ms
step:731/1480 train_time:110174ms step_avg:152.81ms
step:732/1480 train_time:110334ms step_avg:152.82ms
step:733/1480 train_time:110496ms step_avg:152.83ms
step:734/1480 train_time:110658ms step_avg:152.84ms
step:735/1480 train_time:110820ms step_avg:152.85ms
step:736/1480 train_time:110984ms step_avg:152.87ms
step:737/1480 train_time:111146ms step_avg:152.88ms
step:738/1480 train_time:111309ms step_avg:152.90ms
step:739/1480 train_time:111470ms step_avg:152.91ms
step:740/1480 train_time:111635ms step_avg:152.92ms
step:741/1480 train_time:111796ms step_avg:152.94ms
step:742/1480 train_time:111958ms step_avg:152.95ms
step:743/1480 train_time:112118ms step_avg:152.96ms
step:744/1480 train_time:112282ms step_avg:152.97ms
step:745/1480 train_time:112447ms step_avg:152.99ms
step:746/1480 train_time:112609ms step_avg:153.00ms
step:747/1480 train_time:112769ms step_avg:153.01ms
step:748/1480 train_time:112933ms step_avg:153.03ms
step:749/1480 train_time:113096ms step_avg:153.04ms
step:750/1480 train_time:113255ms step_avg:153.05ms
step:750/1480 val_loss:3.5506 train_time:113318ms step_avg:153.13ms
step:751/1480 train_time:113419ms step_avg:153.06ms
step:752/1480 train_time:113581ms step_avg:153.07ms
step:753/1480 train_time:113744ms step_avg:153.09ms
step:754/1480 train_time:113905ms step_avg:153.10ms
step:755/1480 train_time:114066ms step_avg:153.11ms
step:756/1480 train_time:114227ms step_avg:153.12ms
step:757/1480 train_time:114394ms step_avg:153.14ms
step:758/1480 train_time:114555ms step_avg:153.15ms
step:759/1480 train_time:114715ms step_avg:153.16ms
step:760/1480 train_time:114877ms step_avg:153.17ms
step:761/1480 train_time:115039ms step_avg:153.18ms
step:762/1480 train_time:115200ms step_avg:153.19ms
step:763/1480 train_time:115362ms step_avg:153.20ms
step:764/1480 train_time:115523ms step_avg:153.21ms
step:765/1480 train_time:115684ms step_avg:153.22ms
step:766/1480 train_time:115849ms step_avg:153.24ms
step:767/1480 train_time:116012ms step_avg:153.25ms
step:768/1480 train_time:116174ms step_avg:153.26ms
step:769/1480 train_time:116337ms step_avg:153.28ms
step:770/1480 train_time:116499ms step_avg:153.29ms
step:771/1480 train_time:116663ms step_avg:153.30ms
step:772/1480 train_time:116827ms step_avg:153.32ms
step:773/1480 train_time:116989ms step_avg:153.33ms
step:774/1480 train_time:117154ms step_avg:153.34ms
step:775/1480 train_time:117316ms step_avg:153.35ms
step:776/1480 train_time:117480ms step_avg:153.37ms
step:777/1480 train_time:117645ms step_avg:153.38ms
step:778/1480 train_time:117810ms step_avg:153.40ms
step:779/1480 train_time:117974ms step_avg:153.41ms
step:780/1480 train_time:118136ms step_avg:153.42ms
step:781/1480 train_time:118299ms step_avg:153.44ms
step:782/1480 train_time:118461ms step_avg:153.45ms
step:783/1480 train_time:118622ms step_avg:153.46ms
step:784/1480 train_time:118787ms step_avg:153.47ms
step:785/1480 train_time:118950ms step_avg:153.48ms
step:786/1480 train_time:119114ms step_avg:153.50ms
step:787/1480 train_time:119277ms step_avg:153.51ms
step:788/1480 train_time:119440ms step_avg:153.52ms
step:789/1480 train_time:119601ms step_avg:153.53ms
step:790/1480 train_time:119768ms step_avg:153.55ms
step:791/1480 train_time:119934ms step_avg:153.56ms
step:792/1480 train_time:120098ms step_avg:153.58ms
step:793/1480 train_time:120259ms step_avg:153.59ms
step:794/1480 train_time:120424ms step_avg:153.60ms
step:795/1480 train_time:120592ms step_avg:153.62ms
step:796/1480 train_time:120757ms step_avg:153.64ms
step:797/1480 train_time:120921ms step_avg:153.65ms
step:798/1480 train_time:121086ms step_avg:153.66ms
step:799/1480 train_time:121254ms step_avg:153.68ms
step:800/1480 train_time:121416ms step_avg:153.69ms
step:801/1480 train_time:121578ms step_avg:153.70ms
step:802/1480 train_time:121745ms step_avg:153.72ms
step:803/1480 train_time:121909ms step_avg:153.73ms
step:804/1480 train_time:122072ms step_avg:153.74ms
step:805/1480 train_time:122236ms step_avg:153.76ms
step:806/1480 train_time:122397ms step_avg:153.76ms
step:807/1480 train_time:122558ms step_avg:153.77ms
step:808/1480 train_time:122724ms step_avg:153.79ms
step:809/1480 train_time:122887ms step_avg:153.80ms
step:810/1480 train_time:123052ms step_avg:153.81ms
step:811/1480 train_time:123213ms step_avg:153.82ms
step:812/1480 train_time:123376ms step_avg:153.84ms
step:813/1480 train_time:123536ms step_avg:153.84ms
step:814/1480 train_time:123698ms step_avg:153.85ms
step:815/1480 train_time:123861ms step_avg:153.87ms
step:816/1480 train_time:124029ms step_avg:153.88ms
step:817/1480 train_time:124192ms step_avg:153.89ms
step:818/1480 train_time:124354ms step_avg:153.90ms
step:819/1480 train_time:124517ms step_avg:153.91ms
step:820/1480 train_time:124680ms step_avg:153.93ms
step:821/1480 train_time:124842ms step_avg:153.94ms
step:822/1480 train_time:125006ms step_avg:153.95ms
step:823/1480 train_time:125169ms step_avg:153.96ms
step:824/1480 train_time:125332ms step_avg:153.97ms
step:825/1480 train_time:125497ms step_avg:153.98ms
step:826/1480 train_time:125662ms step_avg:154.00ms
step:827/1480 train_time:125827ms step_avg:154.01ms
step:828/1480 train_time:125989ms step_avg:154.02ms
step:829/1480 train_time:126154ms step_avg:154.03ms
step:830/1480 train_time:126318ms step_avg:154.05ms
step:831/1480 train_time:126481ms step_avg:154.06ms
step:832/1480 train_time:126645ms step_avg:154.07ms
step:833/1480 train_time:126810ms step_avg:154.08ms
step:834/1480 train_time:126975ms step_avg:154.10ms
step:835/1480 train_time:127137ms step_avg:154.11ms
step:836/1480 train_time:127302ms step_avg:154.12ms
step:837/1480 train_time:127467ms step_avg:154.13ms
step:838/1480 train_time:127632ms step_avg:154.15ms
step:839/1480 train_time:127794ms step_avg:154.15ms
step:840/1480 train_time:127954ms step_avg:154.16ms
step:841/1480 train_time:128115ms step_avg:154.17ms
step:842/1480 train_time:128277ms step_avg:154.18ms
step:843/1480 train_time:128439ms step_avg:154.19ms
step:844/1480 train_time:128601ms step_avg:154.20ms
step:845/1480 train_time:128767ms step_avg:154.21ms
step:846/1480 train_time:128932ms step_avg:154.23ms
step:847/1480 train_time:129096ms step_avg:154.24ms
step:848/1480 train_time:129257ms step_avg:154.24ms
step:849/1480 train_time:129419ms step_avg:154.25ms
step:850/1480 train_time:129582ms step_avg:154.26ms
step:851/1480 train_time:129749ms step_avg:154.28ms
step:852/1480 train_time:129912ms step_avg:154.29ms
step:853/1480 train_time:130075ms step_avg:154.30ms
step:854/1480 train_time:130239ms step_avg:154.31ms
step:855/1480 train_time:130401ms step_avg:154.32ms
step:856/1480 train_time:130564ms step_avg:154.33ms
step:857/1480 train_time:130730ms step_avg:154.35ms
step:858/1480 train_time:130895ms step_avg:154.36ms
step:859/1480 train_time:131058ms step_avg:154.37ms
step:860/1480 train_time:131218ms step_avg:154.37ms
step:861/1480 train_time:131386ms step_avg:154.39ms
step:862/1480 train_time:131555ms step_avg:154.41ms
step:863/1480 train_time:131722ms step_avg:154.42ms
step:864/1480 train_time:131887ms step_avg:154.43ms
step:865/1480 train_time:132050ms step_avg:154.44ms
step:866/1480 train_time:132215ms step_avg:154.46ms
step:867/1480 train_time:132378ms step_avg:154.47ms
step:868/1480 train_time:132540ms step_avg:154.48ms
step:869/1480 train_time:132702ms step_avg:154.48ms
step:870/1480 train_time:132869ms step_avg:154.50ms
step:871/1480 train_time:133033ms step_avg:154.51ms
step:872/1480 train_time:133196ms step_avg:154.52ms
step:873/1480 train_time:133357ms step_avg:154.53ms
step:874/1480 train_time:133523ms step_avg:154.54ms
step:875/1480 train_time:133688ms step_avg:154.55ms
step:875/1480 val_loss:3.5056 train_time:133754ms step_avg:154.63ms
step:876/1480 train_time:133854ms step_avg:154.57ms
step:877/1480 train_time:134019ms step_avg:154.58ms
step:878/1480 train_time:134183ms step_avg:154.59ms
step:879/1480 train_time:134348ms step_avg:154.60ms
step:880/1480 train_time:134511ms step_avg:154.61ms
step:881/1480 train_time:134672ms step_avg:154.62ms
step:882/1480 train_time:134837ms step_avg:154.63ms
step:883/1480 train_time:135004ms step_avg:154.64ms
step:884/1480 train_time:135171ms step_avg:154.66ms
step:885/1480 train_time:135336ms step_avg:154.67ms
step:886/1480 train_time:135503ms step_avg:154.68ms
step:887/1480 train_time:135671ms step_avg:154.70ms
step:888/1480 train_time:135843ms step_avg:154.72ms
step:889/1480 train_time:136011ms step_avg:154.73ms
step:890/1480 train_time:136173ms step_avg:154.74ms
step:891/1480 train_time:136338ms step_avg:154.75ms
step:892/1480 train_time:136504ms step_avg:154.77ms
step:893/1480 train_time:136668ms step_avg:154.78ms
step:894/1480 train_time:136834ms step_avg:154.79ms
step:895/1480 train_time:137003ms step_avg:154.81ms
step:896/1480 train_time:137168ms step_avg:154.82ms
step:897/1480 train_time:137333ms step_avg:154.83ms
step:898/1480 train_time:137498ms step_avg:154.84ms
step:899/1480 train_time:137663ms step_avg:154.85ms
step:900/1480 train_time:137828ms step_avg:154.86ms
step:901/1480 train_time:137991ms step_avg:154.87ms
step:902/1480 train_time:138153ms step_avg:154.88ms
step:903/1480 train_time:138326ms step_avg:154.90ms
step:904/1480 train_time:138491ms step_avg:154.91ms
step:905/1480 train_time:138652ms step_avg:154.92ms
step:906/1480 train_time:138817ms step_avg:154.93ms
step:907/1480 train_time:138985ms step_avg:154.94ms
step:908/1480 train_time:139149ms step_avg:154.95ms
step:909/1480 train_time:139313ms step_avg:154.96ms
step:910/1480 train_time:139486ms step_avg:154.98ms
step:911/1480 train_time:139652ms step_avg:155.00ms
step:912/1480 train_time:139819ms step_avg:155.01ms
step:913/1480 train_time:139988ms step_avg:155.03ms
step:914/1480 train_time:140156ms step_avg:155.04ms
step:915/1480 train_time:140327ms step_avg:155.06ms
step:916/1480 train_time:140491ms step_avg:155.07ms
step:917/1480 train_time:140655ms step_avg:155.08ms
step:918/1480 train_time:140824ms step_avg:155.09ms
step:919/1480 train_time:140994ms step_avg:155.11ms
step:920/1480 train_time:141159ms step_avg:155.12ms
step:921/1480 train_time:141324ms step_avg:155.13ms
step:922/1480 train_time:141491ms step_avg:155.14ms
step:923/1480 train_time:141652ms step_avg:155.15ms
step:924/1480 train_time:141817ms step_avg:155.16ms
step:925/1480 train_time:141983ms step_avg:155.17ms
step:926/1480 train_time:142148ms step_avg:155.18ms
step:927/1480 train_time:142311ms step_avg:155.19ms
step:928/1480 train_time:142476ms step_avg:155.20ms
step:929/1480 train_time:142643ms step_avg:155.22ms
step:930/1480 train_time:142810ms step_avg:155.23ms
step:931/1480 train_time:142973ms step_avg:155.24ms
step:932/1480 train_time:143139ms step_avg:155.25ms
step:933/1480 train_time:143308ms step_avg:155.26ms
step:934/1480 train_time:143473ms step_avg:155.27ms
step:935/1480 train_time:143644ms step_avg:155.29ms
step:936/1480 train_time:143812ms step_avg:155.30ms
step:937/1480 train_time:143982ms step_avg:155.32ms
step:938/1480 train_time:144147ms step_avg:155.33ms
step:939/1480 train_time:144316ms step_avg:155.35ms
step:940/1480 train_time:144484ms step_avg:155.36ms
step:941/1480 train_time:144648ms step_avg:155.37ms
step:942/1480 train_time:144813ms step_avg:155.38ms
step:943/1480 train_time:144985ms step_avg:155.40ms
step:944/1480 train_time:145157ms step_avg:155.41ms
step:945/1480 train_time:145320ms step_avg:155.42ms
step:946/1480 train_time:145491ms step_avg:155.44ms
step:947/1480 train_time:145658ms step_avg:155.45ms
step:948/1480 train_time:145823ms step_avg:155.46ms
step:949/1480 train_time:145990ms step_avg:155.47ms
step:950/1480 train_time:146154ms step_avg:155.48ms
step:951/1480 train_time:146324ms step_avg:155.50ms
step:952/1480 train_time:146490ms step_avg:155.51ms
step:953/1480 train_time:146658ms step_avg:155.52ms
step:954/1480 train_time:146828ms step_avg:155.54ms
step:955/1480 train_time:146992ms step_avg:155.55ms
step:956/1480 train_time:147157ms step_avg:155.56ms
step:957/1480 train_time:147326ms step_avg:155.57ms
step:958/1480 train_time:147495ms step_avg:155.59ms
step:959/1480 train_time:147660ms step_avg:155.60ms
step:960/1480 train_time:147828ms step_avg:155.61ms
step:961/1480 train_time:147993ms step_avg:155.62ms
step:962/1480 train_time:148156ms step_avg:155.63ms
step:963/1480 train_time:148323ms step_avg:155.64ms
step:964/1480 train_time:148492ms step_avg:155.65ms
step:965/1480 train_time:148656ms step_avg:155.66ms
step:966/1480 train_time:148822ms step_avg:155.67ms
step:967/1480 train_time:148987ms step_avg:155.68ms
step:968/1480 train_time:149151ms step_avg:155.69ms
step:969/1480 train_time:149316ms step_avg:155.70ms
step:970/1480 train_time:149478ms step_avg:155.71ms
step:971/1480 train_time:149643ms step_avg:155.72ms
step:972/1480 train_time:149809ms step_avg:155.73ms
step:973/1480 train_time:149972ms step_avg:155.73ms
step:974/1480 train_time:150142ms step_avg:155.75ms
step:975/1480 train_time:150310ms step_avg:155.76ms
step:976/1480 train_time:150475ms step_avg:155.77ms
step:977/1480 train_time:150639ms step_avg:155.78ms
step:978/1480 train_time:150807ms step_avg:155.79ms
step:979/1480 train_time:150973ms step_avg:155.80ms
step:980/1480 train_time:151140ms step_avg:155.81ms
step:981/1480 train_time:151308ms step_avg:155.83ms
step:982/1480 train_time:151471ms step_avg:155.83ms
step:983/1480 train_time:151636ms step_avg:155.84ms
step:984/1480 train_time:151802ms step_avg:155.85ms
step:985/1480 train_time:151968ms step_avg:155.86ms
step:986/1480 train_time:152133ms step_avg:155.87ms
step:987/1480 train_time:152296ms step_avg:155.88ms
step:988/1480 train_time:152464ms step_avg:155.89ms
step:989/1480 train_time:152631ms step_avg:155.90ms
step:990/1480 train_time:152798ms step_avg:155.92ms
step:991/1480 train_time:152965ms step_avg:155.93ms
step:992/1480 train_time:153139ms step_avg:155.95ms
step:993/1480 train_time:153315ms step_avg:155.97ms
step:994/1480 train_time:153480ms step_avg:155.98ms
step:995/1480 train_time:153645ms step_avg:155.98ms
step:996/1480 train_time:153809ms step_avg:155.99ms
step:997/1480 train_time:153972ms step_avg:156.00ms
step:998/1480 train_time:154136ms step_avg:156.01ms
step:999/1480 train_time:154304ms step_avg:156.02ms
step:1000/1480 train_time:154471ms step_avg:156.03ms
step:1000/1480 val_loss:3.4416 train_time:154539ms step_avg:156.10ms
step:1001/1480 train_time:154641ms step_avg:156.05ms
step:1002/1480 train_time:154806ms step_avg:156.05ms
step:1003/1480 train_time:154978ms step_avg:156.07ms
step:1004/1480 train_time:155146ms step_avg:156.08ms
step:1005/1480 train_time:155312ms step_avg:156.09ms
step:1006/1480 train_time:155481ms step_avg:156.11ms
step:1007/1480 train_time:155645ms step_avg:156.11ms
step:1008/1480 train_time:155811ms step_avg:156.12ms
step:1009/1480 train_time:155984ms step_avg:156.14ms
step:1010/1480 train_time:156149ms step_avg:156.15ms
step:1011/1480 train_time:156315ms step_avg:156.16ms
step:1012/1480 train_time:156481ms step_avg:156.17ms
step:1013/1480 train_time:156651ms step_avg:156.18ms
step:1014/1480 train_time:156819ms step_avg:156.19ms
step:1015/1480 train_time:156989ms step_avg:156.21ms
step:1016/1480 train_time:157158ms step_avg:156.22ms
step:1017/1480 train_time:157329ms step_avg:156.24ms
step:1018/1480 train_time:157497ms step_avg:156.25ms
step:1019/1480 train_time:157665ms step_avg:156.26ms
step:1020/1480 train_time:157834ms step_avg:156.27ms
step:1021/1480 train_time:158000ms step_avg:156.28ms
step:1022/1480 train_time:158167ms step_avg:156.29ms
step:1023/1480 train_time:158334ms step_avg:156.30ms
step:1024/1480 train_time:158503ms step_avg:156.31ms
step:1025/1480 train_time:158673ms step_avg:156.33ms
step:1026/1480 train_time:158839ms step_avg:156.34ms
step:1027/1480 train_time:159007ms step_avg:156.35ms
step:1028/1480 train_time:159179ms step_avg:156.36ms
step:1029/1480 train_time:159354ms step_avg:156.38ms
step:1030/1480 train_time:159523ms step_avg:156.40ms
step:1031/1480 train_time:159687ms step_avg:156.40ms
step:1032/1480 train_time:159862ms step_avg:156.42ms
step:1033/1480 train_time:160028ms step_avg:156.43ms
step:1034/1480 train_time:160196ms step_avg:156.44ms
step:1035/1480 train_time:160365ms step_avg:156.45ms
step:1036/1480 train_time:160529ms step_avg:156.46ms
step:1037/1480 train_time:160696ms step_avg:156.47ms
step:1038/1480 train_time:160864ms step_avg:156.48ms
step:1039/1480 train_time:161034ms step_avg:156.50ms
step:1040/1480 train_time:161202ms step_avg:156.51ms
step:1041/1480 train_time:161370ms step_avg:156.52ms
step:1042/1480 train_time:161534ms step_avg:156.52ms
step:1043/1480 train_time:161700ms step_avg:156.53ms
step:1044/1480 train_time:161866ms step_avg:156.54ms
step:1045/1480 train_time:162034ms step_avg:156.55ms
step:1046/1480 train_time:162204ms step_avg:156.57ms
step:1047/1480 train_time:162370ms step_avg:156.58ms
step:1048/1480 train_time:162536ms step_avg:156.59ms
step:1049/1480 train_time:162703ms step_avg:156.60ms
step:1050/1480 train_time:162870ms step_avg:156.61ms
step:1051/1480 train_time:163040ms step_avg:156.62ms
step:1052/1480 train_time:163208ms step_avg:156.63ms
step:1053/1480 train_time:163374ms step_avg:156.64ms
step:1054/1480 train_time:163544ms step_avg:156.65ms
step:1055/1480 train_time:163709ms step_avg:156.66ms
step:1056/1480 train_time:163874ms step_avg:156.67ms
step:1057/1480 train_time:164043ms step_avg:156.68ms
step:1058/1480 train_time:164211ms step_avg:156.69ms
step:1059/1480 train_time:164385ms step_avg:156.71ms
step:1060/1480 train_time:164553ms step_avg:156.72ms
step:1061/1480 train_time:164716ms step_avg:156.72ms
step:1062/1480 train_time:164884ms step_avg:156.73ms
step:1063/1480 train_time:165048ms step_avg:156.74ms
step:1064/1480 train_time:165211ms step_avg:156.75ms
step:1065/1480 train_time:165378ms step_avg:156.76ms
step:1066/1480 train_time:165546ms step_avg:156.77ms
step:1067/1480 train_time:165713ms step_avg:156.78ms
step:1068/1480 train_time:165878ms step_avg:156.78ms
step:1069/1480 train_time:166049ms step_avg:156.80ms
step:1070/1480 train_time:166214ms step_avg:156.81ms
step:1071/1480 train_time:166387ms step_avg:156.82ms
step:1072/1480 train_time:166553ms step_avg:156.83ms
step:1073/1480 train_time:166718ms step_avg:156.84ms
step:1074/1480 train_time:166885ms step_avg:156.85ms
step:1075/1480 train_time:167056ms step_avg:156.86ms
step:1076/1480 train_time:167225ms step_avg:156.87ms
step:1077/1480 train_time:167390ms step_avg:156.88ms
step:1078/1480 train_time:167567ms step_avg:156.90ms
step:1079/1480 train_time:167740ms step_avg:156.91ms
step:1080/1480 train_time:167910ms step_avg:156.93ms
step:1081/1480 train_time:168077ms step_avg:156.93ms
step:1082/1480 train_time:168244ms step_avg:156.94ms
step:1083/1480 train_time:168410ms step_avg:156.95ms
step:1084/1480 train_time:168578ms step_avg:156.96ms
step:1085/1480 train_time:168748ms step_avg:156.97ms
step:1086/1480 train_time:168916ms step_avg:156.98ms
step:1087/1480 train_time:169083ms step_avg:156.99ms
step:1088/1480 train_time:169251ms step_avg:157.01ms
step:1089/1480 train_time:169424ms step_avg:157.02ms
step:1090/1480 train_time:169594ms step_avg:157.03ms
step:1091/1480 train_time:169764ms step_avg:157.04ms
step:1092/1480 train_time:169930ms step_avg:157.05ms
step:1093/1480 train_time:170099ms step_avg:157.06ms
step:1094/1480 train_time:170265ms step_avg:157.07ms
step:1095/1480 train_time:170429ms step_avg:157.08ms
step:1096/1480 train_time:170598ms step_avg:157.09ms
step:1097/1480 train_time:170766ms step_avg:157.10ms
step:1098/1480 train_time:170936ms step_avg:157.11ms
step:1099/1480 train_time:171108ms step_avg:157.12ms
step:1100/1480 train_time:171282ms step_avg:157.14ms
step:1101/1480 train_time:171451ms step_avg:157.15ms
step:1102/1480 train_time:171624ms step_avg:157.16ms
step:1103/1480 train_time:171799ms step_avg:157.18ms
step:1104/1480 train_time:171968ms step_avg:157.19ms
step:1105/1480 train_time:172136ms step_avg:157.20ms
step:1106/1480 train_time:172304ms step_avg:157.21ms
step:1107/1480 train_time:172472ms step_avg:157.22ms
step:1108/1480 train_time:172637ms step_avg:157.23ms
step:1109/1480 train_time:172803ms step_avg:157.24ms
step:1110/1480 train_time:172969ms step_avg:157.24ms
step:1111/1480 train_time:173137ms step_avg:157.25ms
step:1112/1480 train_time:173306ms step_avg:157.27ms
step:1113/1480 train_time:173486ms step_avg:157.29ms
step:1114/1480 train_time:173658ms step_avg:157.30ms
step:1115/1480 train_time:173831ms step_avg:157.31ms
step:1116/1480 train_time:173996ms step_avg:157.32ms
step:1117/1480 train_time:174170ms step_avg:157.33ms
step:1118/1480 train_time:174344ms step_avg:157.35ms
step:1119/1480 train_time:174511ms step_avg:157.36ms
step:1120/1480 train_time:174681ms step_avg:157.37ms
step:1121/1480 train_time:174850ms step_avg:157.38ms
step:1122/1480 train_time:175017ms step_avg:157.39ms
step:1123/1480 train_time:175184ms step_avg:157.40ms
step:1124/1480 train_time:175353ms step_avg:157.41ms
step:1125/1480 train_time:175521ms step_avg:157.42ms
step:1125/1480 val_loss:3.3864 train_time:175589ms step_avg:157.48ms
step:1126/1480 train_time:175693ms step_avg:157.43ms
step:1127/1480 train_time:175862ms step_avg:157.44ms
step:1128/1480 train_time:176032ms step_avg:157.45ms
step:1129/1480 train_time:176205ms step_avg:157.47ms
step:1130/1480 train_time:176375ms step_avg:157.48ms
step:1131/1480 train_time:176550ms step_avg:157.49ms
step:1132/1480 train_time:176715ms step_avg:157.50ms
step:1133/1480 train_time:176886ms step_avg:157.51ms
step:1134/1480 train_time:177057ms step_avg:157.52ms
step:1135/1480 train_time:177224ms step_avg:157.53ms
step:1136/1480 train_time:177395ms step_avg:157.54ms
step:1137/1480 train_time:177563ms step_avg:157.55ms
step:1138/1480 train_time:177734ms step_avg:157.57ms
step:1139/1480 train_time:177903ms step_avg:157.58ms
step:1140/1480 train_time:178070ms step_avg:157.58ms
step:1141/1480 train_time:178241ms step_avg:157.60ms
step:1142/1480 train_time:178409ms step_avg:157.61ms
step:1143/1480 train_time:178579ms step_avg:157.62ms
step:1144/1480 train_time:178748ms step_avg:157.63ms
step:1145/1480 train_time:178915ms step_avg:157.63ms
step:1146/1480 train_time:179086ms step_avg:157.65ms
step:1147/1480 train_time:179255ms step_avg:157.66ms
step:1148/1480 train_time:179423ms step_avg:157.67ms
step:1149/1480 train_time:179595ms step_avg:157.68ms
step:1150/1480 train_time:179763ms step_avg:157.69ms
step:1151/1480 train_time:179934ms step_avg:157.70ms
step:1152/1480 train_time:180105ms step_avg:157.71ms
step:1153/1480 train_time:180279ms step_avg:157.72ms
step:1154/1480 train_time:180445ms step_avg:157.73ms
step:1155/1480 train_time:180619ms step_avg:157.75ms
step:1156/1480 train_time:180798ms step_avg:157.76ms
step:1157/1480 train_time:180967ms step_avg:157.77ms
step:1158/1480 train_time:181135ms step_avg:157.78ms
step:1159/1480 train_time:181301ms step_avg:157.79ms
step:1160/1480 train_time:181467ms step_avg:157.80ms
step:1161/1480 train_time:181638ms step_avg:157.81ms
step:1162/1480 train_time:181808ms step_avg:157.82ms
step:1163/1480 train_time:181978ms step_avg:157.83ms
step:1164/1480 train_time:182147ms step_avg:157.84ms
step:1165/1480 train_time:182313ms step_avg:157.85ms
step:1166/1480 train_time:182481ms step_avg:157.86ms
step:1167/1480 train_time:182648ms step_avg:157.86ms
step:1168/1480 train_time:182816ms step_avg:157.87ms
step:1169/1480 train_time:182984ms step_avg:157.88ms
step:1170/1480 train_time:183153ms step_avg:157.89ms
step:1171/1480 train_time:183319ms step_avg:157.90ms
step:1172/1480 train_time:183488ms step_avg:157.91ms
step:1173/1480 train_time:183658ms step_avg:157.92ms
step:1174/1480 train_time:183839ms step_avg:157.94ms
step:1175/1480 train_time:184012ms step_avg:157.95ms
step:1176/1480 train_time:184182ms step_avg:157.96ms
step:1177/1480 train_time:184359ms step_avg:157.98ms
step:1178/1480 train_time:184525ms step_avg:157.98ms
step:1179/1480 train_time:184692ms step_avg:157.99ms
step:1180/1480 train_time:184871ms step_avg:158.01ms
step:1181/1480 train_time:185040ms step_avg:158.02ms
step:1182/1480 train_time:185207ms step_avg:158.03ms
step:1183/1480 train_time:185378ms step_avg:158.04ms
step:1184/1480 train_time:185547ms step_avg:158.05ms
step:1185/1480 train_time:185719ms step_avg:158.06ms
step:1186/1480 train_time:185890ms step_avg:158.07ms
step:1187/1480 train_time:186075ms step_avg:158.09ms
step:1188/1480 train_time:186242ms step_avg:158.10ms
step:1189/1480 train_time:186413ms step_avg:158.11ms
step:1190/1480 train_time:186580ms step_avg:158.12ms
step:1191/1480 train_time:186753ms step_avg:158.13ms
step:1192/1480 train_time:186919ms step_avg:158.14ms
step:1193/1480 train_time:187085ms step_avg:158.14ms
step:1194/1480 train_time:187256ms step_avg:158.16ms
step:1195/1480 train_time:187428ms step_avg:158.17ms
step:1196/1480 train_time:187613ms step_avg:158.19ms
step:1197/1480 train_time:187784ms step_avg:158.20ms
step:1198/1480 train_time:187967ms step_avg:158.22ms
step:1199/1480 train_time:188137ms step_avg:158.23ms
step:1200/1480 train_time:188303ms step_avg:158.24ms
step:1201/1480 train_time:188473ms step_avg:158.25ms
step:1202/1480 train_time:188655ms step_avg:158.27ms
step:1203/1480 train_time:188833ms step_avg:158.28ms
step:1204/1480 train_time:189007ms step_avg:158.30ms
step:1205/1480 train_time:189175ms step_avg:158.31ms
step:1206/1480 train_time:189340ms step_avg:158.31ms
step:1207/1480 train_time:189509ms step_avg:158.32ms
step:1208/1480 train_time:189677ms step_avg:158.33ms
step:1209/1480 train_time:189848ms step_avg:158.34ms
step:1210/1480 train_time:190022ms step_avg:158.35ms
step:1211/1480 train_time:190198ms step_avg:158.37ms
step:1212/1480 train_time:190368ms step_avg:158.38ms
step:1213/1480 train_time:190540ms step_avg:158.39ms
step:1214/1480 train_time:190717ms step_avg:158.40ms
step:1215/1480 train_time:190891ms step_avg:158.42ms
step:1216/1480 train_time:191060ms step_avg:158.42ms
step:1217/1480 train_time:191235ms step_avg:158.44ms
step:1218/1480 train_time:191404ms step_avg:158.45ms
step:1219/1480 train_time:191583ms step_avg:158.46ms
step:1220/1480 train_time:191753ms step_avg:158.47ms
step:1221/1480 train_time:191922ms step_avg:158.48ms
step:1222/1480 train_time:192091ms step_avg:158.49ms
step:1223/1480 train_time:192260ms step_avg:158.50ms
step:1224/1480 train_time:192438ms step_avg:158.52ms
step:1225/1480 train_time:192610ms step_avg:158.53ms
step:1226/1480 train_time:192783ms step_avg:158.54ms
step:1227/1480 train_time:192957ms step_avg:158.55ms
step:1228/1480 train_time:193127ms step_avg:158.56ms
step:1229/1480 train_time:193300ms step_avg:158.57ms
step:1230/1480 train_time:193480ms step_avg:158.59ms
step:1231/1480 train_time:193656ms step_avg:158.60ms
step:1232/1480 train_time:193829ms step_avg:158.62ms
step:1233/1480 train_time:193999ms step_avg:158.63ms
step:1234/1480 train_time:194169ms step_avg:158.63ms
step:1235/1480 train_time:194342ms step_avg:158.65ms
step:1236/1480 train_time:194511ms step_avg:158.66ms
step:1237/1480 train_time:194684ms step_avg:158.67ms
step:1238/1480 train_time:194870ms step_avg:158.69ms
step:1239/1480 train_time:195040ms step_avg:158.70ms
step:1240/1480 train_time:195211ms step_avg:158.71ms
step:1241/1480 train_time:195383ms step_avg:158.72ms
step:1242/1480 train_time:195554ms step_avg:158.73ms
step:1243/1480 train_time:195729ms step_avg:158.74ms
step:1244/1480 train_time:195896ms step_avg:158.75ms
step:1245/1480 train_time:196063ms step_avg:158.76ms
step:1246/1480 train_time:196234ms step_avg:158.77ms
step:1247/1480 train_time:196401ms step_avg:158.77ms
step:1248/1480 train_time:196571ms step_avg:158.78ms
step:1249/1480 train_time:196738ms step_avg:158.79ms
step:1250/1480 train_time:196907ms step_avg:158.80ms
step:1250/1480 val_loss:3.3363 train_time:196979ms step_avg:158.85ms
step:1251/1480 train_time:197088ms step_avg:158.81ms
step:1252/1480 train_time:197257ms step_avg:158.82ms
step:1253/1480 train_time:197424ms step_avg:158.83ms
step:1254/1480 train_time:197594ms step_avg:158.84ms
step:1255/1480 train_time:197782ms step_avg:158.86ms
step:1256/1480 train_time:197956ms step_avg:158.87ms
step:1257/1480 train_time:198126ms step_avg:158.88ms
step:1258/1480 train_time:198302ms step_avg:158.90ms
step:1259/1480 train_time:198474ms step_avg:158.91ms
step:1260/1480 train_time:198641ms step_avg:158.91ms
step:1261/1480 train_time:198811ms step_avg:158.92ms
step:1262/1480 train_time:198986ms step_avg:158.93ms
step:1263/1480 train_time:199160ms step_avg:158.95ms
step:1264/1480 train_time:199326ms step_avg:158.95ms
step:1265/1480 train_time:199495ms step_avg:158.96ms
step:1266/1480 train_time:199667ms step_avg:158.97ms
step:1267/1480 train_time:199838ms step_avg:158.98ms
step:1268/1480 train_time:200010ms step_avg:158.99ms
step:1269/1480 train_time:200186ms step_avg:159.00ms
step:1270/1480 train_time:200356ms step_avg:159.01ms
step:1271/1480 train_time:200526ms step_avg:159.02ms
step:1272/1480 train_time:200693ms step_avg:159.03ms
step:1273/1480 train_time:200862ms step_avg:159.04ms
step:1274/1480 train_time:201034ms step_avg:159.05ms
step:1275/1480 train_time:201204ms step_avg:159.05ms
step:1276/1480 train_time:201368ms step_avg:159.06ms
step:1277/1480 train_time:201543ms step_avg:159.07ms
step:1278/1480 train_time:201711ms step_avg:159.08ms
step:1279/1480 train_time:201883ms step_avg:159.09ms
step:1280/1480 train_time:202062ms step_avg:159.10ms
step:1281/1480 train_time:202231ms step_avg:159.11ms
step:1282/1480 train_time:202399ms step_avg:159.12ms
step:1283/1480 train_time:202567ms step_avg:159.13ms
step:1284/1480 train_time:202738ms step_avg:159.13ms
step:1285/1480 train_time:202907ms step_avg:159.14ms
step:1286/1480 train_time:203077ms step_avg:159.15ms
step:1287/1480 train_time:203248ms step_avg:159.16ms
step:1288/1480 train_time:203421ms step_avg:159.17ms
step:1289/1480 train_time:203602ms step_avg:159.19ms
step:1290/1480 train_time:203781ms step_avg:159.20ms
step:1291/1480 train_time:203954ms step_avg:159.21ms
step:1292/1480 train_time:204128ms step_avg:159.23ms
step:1293/1480 train_time:204303ms step_avg:159.24ms
step:1294/1480 train_time:204474ms step_avg:159.25ms
step:1295/1480 train_time:204645ms step_avg:159.26ms
step:1296/1480 train_time:204819ms step_avg:159.27ms
step:1297/1480 train_time:204989ms step_avg:159.28ms
step:1298/1480 train_time:205160ms step_avg:159.29ms
step:1299/1480 train_time:205330ms step_avg:159.29ms
step:1300/1480 train_time:205499ms step_avg:159.30ms
step:1301/1480 train_time:205665ms step_avg:159.31ms
step:1302/1480 train_time:205841ms step_avg:159.32ms
step:1303/1480 train_time:206018ms step_avg:159.33ms
step:1304/1480 train_time:206192ms step_avg:159.34ms
step:1305/1480 train_time:206361ms step_avg:159.35ms
step:1306/1480 train_time:206535ms step_avg:159.36ms
step:1307/1480 train_time:206704ms step_avg:159.37ms
step:1308/1480 train_time:206873ms step_avg:159.38ms
step:1309/1480 train_time:207048ms step_avg:159.39ms
step:1310/1480 train_time:207219ms step_avg:159.40ms
step:1311/1480 train_time:207387ms step_avg:159.41ms
step:1312/1480 train_time:207560ms step_avg:159.42ms
step:1313/1480 train_time:207728ms step_avg:159.42ms
step:1314/1480 train_time:207903ms step_avg:159.43ms
step:1315/1480 train_time:208073ms step_avg:159.44ms
step:1316/1480 train_time:208241ms step_avg:159.45ms
step:1317/1480 train_time:208413ms step_avg:159.46ms
step:1318/1480 train_time:208595ms step_avg:159.48ms
step:1319/1480 train_time:208770ms step_avg:159.49ms
step:1320/1480 train_time:208948ms step_avg:159.50ms
step:1321/1480 train_time:209122ms step_avg:159.51ms
step:1322/1480 train_time:209301ms step_avg:159.53ms
step:1323/1480 train_time:209472ms step_avg:159.54ms
step:1324/1480 train_time:209648ms step_avg:159.55ms
step:1325/1480 train_time:209829ms step_avg:159.57ms
step:1326/1480 train_time:210006ms step_avg:159.58ms
step:1327/1480 train_time:210176ms step_avg:159.59ms
step:1328/1480 train_time:210346ms step_avg:159.60ms
step:1329/1480 train_time:210542ms step_avg:159.62ms
step:1330/1480 train_time:210722ms step_avg:159.64ms
step:1331/1480 train_time:210893ms step_avg:159.65ms
step:1332/1480 train_time:211068ms step_avg:159.66ms
step:1333/1480 train_time:211245ms step_avg:159.67ms
step:1334/1480 train_time:211417ms step_avg:159.68ms
step:1335/1480 train_time:211587ms step_avg:159.69ms
step:1336/1480 train_time:211772ms step_avg:159.71ms
step:1337/1480 train_time:211949ms step_avg:159.72ms
step:1338/1480 train_time:212122ms step_avg:159.73ms
step:1339/1480 train_time:212296ms step_avg:159.74ms
step:1340/1480 train_time:212468ms step_avg:159.75ms
step:1341/1480 train_time:212637ms step_avg:159.76ms
step:1342/1480 train_time:212810ms step_avg:159.77ms
step:1343/1480 train_time:212982ms step_avg:159.78ms
step:1344/1480 train_time:213153ms step_avg:159.79ms
step:1345/1480 train_time:213333ms step_avg:159.80ms
step:1346/1480 train_time:213501ms step_avg:159.81ms
step:1347/1480 train_time:213671ms step_avg:159.81ms
step:1348/1480 train_time:213841ms step_avg:159.82ms
step:1349/1480 train_time:214010ms step_avg:159.83ms
step:1350/1480 train_time:214185ms step_avg:159.84ms
step:1351/1480 train_time:214356ms step_avg:159.85ms
step:1352/1480 train_time:214528ms step_avg:159.86ms
step:1353/1480 train_time:214706ms step_avg:159.87ms
step:1354/1480 train_time:214877ms step_avg:159.88ms
step:1355/1480 train_time:215043ms step_avg:159.88ms
step:1356/1480 train_time:215217ms step_avg:159.89ms
step:1357/1480 train_time:215388ms step_avg:159.90ms
step:1358/1480 train_time:215560ms step_avg:159.91ms
step:1359/1480 train_time:215733ms step_avg:159.92ms
step:1360/1480 train_time:215907ms step_avg:159.93ms
step:1361/1480 train_time:216084ms step_avg:159.94ms
step:1362/1480 train_time:216259ms step_avg:159.96ms
step:1363/1480 train_time:216439ms step_avg:159.97ms
step:1364/1480 train_time:216607ms step_avg:159.98ms
step:1365/1480 train_time:216776ms step_avg:159.98ms
step:1366/1480 train_time:216948ms step_avg:159.99ms
step:1367/1480 train_time:217120ms step_avg:160.00ms
step:1368/1480 train_time:217292ms step_avg:160.01ms
step:1369/1480 train_time:217472ms step_avg:160.02ms
step:1370/1480 train_time:217650ms step_avg:160.04ms
step:1371/1480 train_time:217823ms step_avg:160.05ms
step:1372/1480 train_time:218001ms step_avg:160.06ms
step:1373/1480 train_time:218169ms step_avg:160.07ms
step:1374/1480 train_time:218345ms step_avg:160.08ms
step:1375/1480 train_time:218517ms step_avg:160.09ms
step:1375/1480 val_loss:3.2976 train_time:218585ms step_avg:160.14ms
step:1376/1480 train_time:218691ms step_avg:160.10ms
step:1377/1480 train_time:218863ms step_avg:160.10ms
step:1378/1480 train_time:219032ms step_avg:160.11ms
step:1379/1480 train_time:219207ms step_avg:160.12ms
step:1380/1480 train_time:219381ms step_avg:160.13ms
step:1381/1480 train_time:219562ms step_avg:160.15ms
step:1382/1480 train_time:219733ms step_avg:160.16ms
step:1383/1480 train_time:219905ms step_avg:160.16ms
step:1384/1480 train_time:220081ms step_avg:160.18ms
step:1385/1480 train_time:220246ms step_avg:160.18ms
step:1386/1480 train_time:220417ms step_avg:160.19ms
step:1387/1480 train_time:220586ms step_avg:160.19ms
step:1388/1480 train_time:220754ms step_avg:160.20ms
step:1389/1480 train_time:220928ms step_avg:160.21ms
step:1390/1480 train_time:221096ms step_avg:160.21ms
step:1391/1480 train_time:221267ms step_avg:160.22ms
step:1392/1480 train_time:221441ms step_avg:160.23ms
step:1393/1480 train_time:221610ms step_avg:160.24ms
step:1394/1480 train_time:221781ms step_avg:160.25ms
step:1395/1480 train_time:221949ms step_avg:160.25ms
step:1396/1480 train_time:222117ms step_avg:160.26ms
step:1397/1480 train_time:222284ms step_avg:160.26ms
step:1398/1480 train_time:222452ms step_avg:160.27ms
step:1399/1480 train_time:222623ms step_avg:160.28ms
step:1400/1480 train_time:222800ms step_avg:160.29ms
step:1401/1480 train_time:222966ms step_avg:160.29ms
step:1402/1480 train_time:223137ms step_avg:160.30ms
step:1403/1480 train_time:223314ms step_avg:160.31ms
step:1404/1480 train_time:223484ms step_avg:160.32ms
step:1405/1480 train_time:223659ms step_avg:160.33ms
step:1406/1480 train_time:223833ms step_avg:160.34ms
step:1407/1480 train_time:224001ms step_avg:160.34ms
step:1408/1480 train_time:224170ms step_avg:160.35ms
step:1409/1480 train_time:224352ms step_avg:160.37ms
step:1410/1480 train_time:224522ms step_avg:160.37ms
step:1411/1480 train_time:224689ms step_avg:160.38ms
step:1412/1480 train_time:224860ms step_avg:160.39ms
step:1413/1480 train_time:225029ms step_avg:160.39ms
step:1414/1480 train_time:225202ms step_avg:160.40ms
step:1415/1480 train_time:225375ms step_avg:160.41ms
step:1416/1480 train_time:225561ms step_avg:160.43ms
step:1417/1480 train_time:225733ms step_avg:160.44ms
step:1418/1480 train_time:225904ms step_avg:160.44ms
step:1419/1480 train_time:226078ms step_avg:160.45ms
step:1420/1480 train_time:226253ms step_avg:160.46ms
step:1421/1480 train_time:226428ms step_avg:160.47ms
step:1422/1480 train_time:226601ms step_avg:160.48ms
step:1423/1480 train_time:226772ms step_avg:160.49ms
step:1424/1480 train_time:226949ms step_avg:160.50ms
step:1425/1480 train_time:227131ms step_avg:160.52ms
step:1426/1480 train_time:227303ms step_avg:160.52ms
step:1427/1480 train_time:227477ms step_avg:160.53ms
step:1428/1480 train_time:227649ms step_avg:160.54ms
step:1429/1480 train_time:227820ms step_avg:160.55ms
step:1430/1480 train_time:227993ms step_avg:160.56ms
step:1431/1480 train_time:228167ms step_avg:160.57ms
step:1432/1480 train_time:228344ms step_avg:160.58ms
step:1433/1480 train_time:228523ms step_avg:160.59ms
step:1434/1480 train_time:228704ms step_avg:160.61ms
step:1435/1480 train_time:228880ms step_avg:160.62ms
step:1436/1480 train_time:229053ms step_avg:160.63ms
step:1437/1480 train_time:229225ms step_avg:160.63ms
step:1438/1480 train_time:229395ms step_avg:160.64ms
step:1439/1480 train_time:229567ms step_avg:160.65ms
step:1440/1480 train_time:229737ms step_avg:160.66ms
step:1441/1480 train_time:229906ms step_avg:160.66ms
step:1442/1480 train_time:230085ms step_avg:160.67ms
step:1443/1480 train_time:230274ms step_avg:160.69ms
step:1444/1480 train_time:230445ms step_avg:160.70ms
step:1445/1480 train_time:230616ms step_avg:160.71ms
step:1446/1480 train_time:230793ms step_avg:160.72ms
step:1447/1480 train_time:230971ms step_avg:160.73ms
step:1448/1480 train_time:231144ms step_avg:160.74ms
step:1449/1480 train_time:231317ms step_avg:160.75ms
step:1450/1480 train_time:231489ms step_avg:160.76ms
step:1451/1480 train_time:231661ms step_avg:160.76ms
step:1452/1480 train_time:231835ms step_avg:160.77ms
step:1453/1480 train_time:232004ms step_avg:160.78ms
step:1454/1480 train_time:232178ms step_avg:160.79ms
step:1455/1480 train_time:232357ms step_avg:160.80ms
step:1456/1480 train_time:232530ms step_avg:160.81ms
step:1457/1480 train_time:232702ms step_avg:160.82ms
step:1458/1480 train_time:232873ms step_avg:160.82ms
step:1459/1480 train_time:233050ms step_avg:160.84ms
step:1460/1480 train_time:233223ms step_avg:160.84ms
step:1461/1480 train_time:233398ms step_avg:160.85ms
step:1462/1480 train_time:233569ms step_avg:160.86ms
step:1463/1480 train_time:233747ms step_avg:160.87ms
step:1464/1480 train_time:233921ms step_avg:160.88ms
step:1465/1480 train_time:234093ms step_avg:160.89ms
step:1466/1480 train_time:234264ms step_avg:160.90ms
step:1467/1480 train_time:234440ms step_avg:160.91ms
step:1468/1480 train_time:234608ms step_avg:160.91ms
step:1469/1480 train_time:234780ms step_avg:160.92ms
step:1470/1480 train_time:234959ms step_avg:160.93ms
step:1471/1480 train_time:235145ms step_avg:160.95ms
step:1472/1480 train_time:235327ms step_avg:160.96ms
step:1473/1480 train_time:235499ms step_avg:160.97ms
step:1474/1480 train_time:235676ms step_avg:160.98ms
step:1475/1480 train_time:235856ms step_avg:160.99ms
step:1476/1480 train_time:236027ms step_avg:161.00ms
step:1477/1480 train_time:236208ms step_avg:161.01ms
step:1478/1480 train_time:236391ms step_avg:161.03ms
step:1479/1480 train_time:236566ms step_avg:161.04ms
step:1480/1480 train_time:236739ms step_avg:161.05ms
step:1480/1480 val_loss:3.2786 train_time:236810ms step_avg:161.10ms