83fd4151-f2d1-4d8d-ad31-5546543072f3.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
from dataclasses import dataclass
from pathlib import Path

import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
# Use of FlexAttention contributed by @KoszarskyB
from torch.nn.attention.flex_attention import BlockMask, flex_attention

# -----------------------------------------------------------------------------
# Muon optimizer

def zeropower_via_svd(G, steps=None):
    U, S, V = G.svd()
    return U @ V.T

@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert len(G.shape) == 2
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    X /= (X.norm() + eps) # ensure top singular value <= 1
    if G.size(0) > G.size(1):
        X = X.T
    for _ in range(steps):
        A = X @ X.T
        B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    if G.size(0) > G.size(1):
        X = X.T
    return X

zeropower_backends = dict(svd=zeropower_via_svd, newtonschulz5=zeropower_via_newtonschulz5)

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven't tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        backend: The chosen backend for the orthogonalization step. (recommended: 'newtonschulz5')
        backend_steps: The number of iteration steps to use in the backend, if it is iterative.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True,
                 backend='newtonschulz5', backend_steps=5):
        self.num_process = int(os.environ['WORLD_SIZE'])
        self.rank = int(os.environ["RANK"])
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, backend=backend, backend_steps=backend_steps)
        params: "list[torch.Tensor]" = list(params)
        assert all(isinstance(p, torch.Tensor) for p in params)
        sizes = {p.numel() for p in params}
        param_groups = [
            {
                "params": [p for p in params if p.numel() == size],
                "update_buffer": [
                    torch.empty(size, device="cuda", dtype=torch.bfloat16)
                    for _ in range(self.num_process)
                ],
            }
            for size in sizes
        ]
        super().__init__(param_groups, defaults)

    def step(self):
        for group in self.param_groups:
            lr: float = group["lr"]
            momentum: float = group["momentum"]
            nesterov: bool = group["nesterov"]
            zeropower_backend = zeropower_backends[group["backend"]]
            backend_steps: int = group["backend_steps"]
            update_buffers: "list[torch.Tensor]" = group["update_buffer"]
            # generate weight updates in distributed fashion
            params: "list[torch.Tensor]" = group["params"]
            assert len(params) % self.num_process == 0
            handle = None
            params_world = None
            def update_prev():
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffers):
                    p_world.data.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.num_process]:
                p = params[base_i + self.rank]
                g = p.grad
                assert g is not None
                state = self.state[p] 
                if "momentum_buffer" not in state:
                    state["momentum_buffer"] = torch.zeros_like(g)
                buf: torch.Tensor = state["momentum_buffer"]
                buf.lerp_(g, 1 - momentum)
                g = g.lerp_(buf, momentum) if nesterov else buf
                g = zeropower_backend(g, steps=backend_steps).flatten()
                update_prev()
                handle = dist.all_gather(update_buffers, g, async_op=True)
                params_world = params[base_i : base_i + self.num_process]
            update_prev()


# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):

    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=False)

    def forward(self, x):
        return F.linear(x, self.weight.to(x.dtype))

class Rotary(torch.nn.Module):

    def __init__(self, dim, base=10000):
        super().__init__()
        self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None

    def forward(self, x):
        seq_len = x.shape[1]
        if seq_len != self.seq_len_cached:
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.outer(t, self.inv_freq)
            self.seq_len_cached = seq_len
            self.cos_cached = freqs.cos()
            self.sin_cached = freqs.sin()
        cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
        # apply_rotary_emb(x, cos, sin)
        x1, x2 = x.chunk(2, dim=3)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x)

class CausalSelfAttention(nn.Module):

    def __init__(self, dim, n_head):
        super().__init__()
        assert dim % n_head == 0
        self.n_head = n_head
        self.c_q = CastedLinear(dim, dim)
        self.c_k = CastedLinear(dim, dim)
        self.c_v = CastedLinear(dim, dim)
        # value residual lambda
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5])) # @Grad62304977
        # rotary embeddings
        self.rotary = Rotary(dim // n_head) # dim // n_head = head_dim
        # output projection
        self.c_proj = CastedLinear(dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor, vi: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q: torch.Tensor = self.c_q(x).view(B, T, self.n_head, -1)
        k: torch.Tensor = self.c_k(x).view(B, T, self.n_head, -1)
        v: torch.Tensor = self.c_v(x).view(B, T, self.n_head, -1)
        v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @Grad62304977
        q, k = norm(q), norm(k) # QK norm suggested by @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask)
        y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):

    def __init__(self, dim: int):
        super().__init__()
        self.c_fc   = CastedLinear(dim, 4 * dim)
        self.c_proj = CastedLinear(4 * dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):

    def __init__(self, config):
        super().__init__()
        self.attn = CausalSelfAttention(config.n_embd, config.n_head)
        self.mlp = MLP(config.n_embd)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x: torch.Tensor, vi: torch.Tensor, x0: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        x = x + self.attn(norm(x), vi, block_mask)
        x = x + self.mlp(norm(x))
        return x

# -----------------------------------------------------------------------------
# The main GPT-2 model

@dataclass
class GPTConfig:
    vocab_size : int = 50304
    n_layer : int = 12
    n_head : int = 6 # head dim 128 suggested by @Grad62304977
    n_embd : int = 768
    lm_head_softcap : int = 30

class GPT(nn.Module):

    def __init__(self, config: GPTConfig):
        super().__init__()
        self.n_layer = config.n_layer
        self.lm_head_softcap = config.lm_head_softcap

        # U-net design by @brendanh0gan
        self.num_encoder_layers = config.n_layer // 2 # Half of the layers for encoder
        self.num_decoder_layers = config.n_layer - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))

        self.transformer = nn.ModuleDict(dict(
            wte = nn.Embedding(config.vocab_size, config.n_embd),
            # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
            # U-net structure on token value embeddings by @leloykun
            vte = nn.Embedding(config.vocab_size, config.n_embd*self.num_encoder_layers),
            h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
        ))
        self.lm_head = CastedLinear(config.n_embd, config.vocab_size)
        self.lm_head.weight.data.zero_() # @Grad62304977

    def forward(self, idx: torch.Tensor, target: torch.Tensor, sliding_window: torch.Tensor) -> torch.Tensor:
        BLOCK_SIZE = 128
        assert idx.ndim == 1
        docs = (idx == 50256).cumsum(0)
        docs_low = docs.reshape(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.reshape(-1, BLOCK_SIZE)[:, -1].contiguous()
        def document_sliding_window_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            window_mask = q_idx - kv_idx < sliding_window
            return causal_mask & document_mask & window_mask

        S = len(idx)
        def create_sliding_window_causal_mask(S: int, sliding_window: torch.Tensor):
            kv_idx = block_idx = torch.arange(S // BLOCK_SIZE, dtype=torch.int32, device="cuda")
            q_idx = block_idx[:, None]
            causal_mask = q_idx >= kv_idx
            document_mask = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
            window_mask = q_idx - kv_idx < ((sliding_window + BLOCK_SIZE - 1) // BLOCK_SIZE)
            dense_mask = causal_mask & document_mask & window_mask
            dense_mask = dense_mask.to(torch.int32)
            num_blocks = dense_mask.sum(dim=-1).to(torch.int32)
            indices = torch.argsort(dense_mask, dim=-1, descending=True, stable=True).to(torch.int32)
            num_blocks = num_blocks[None, None, :].contiguous()
            indices = indices[None, None, :].contiguous()
            return BlockMask.from_kv_blocks(num_blocks, indices, BLOCK_SIZE=BLOCK_SIZE, mask_mod=document_sliding_window_causal)
        block_mask = create_sliding_window_causal_mask(S, sliding_window)

        # forward the GPT model itself
        x = self.transformer.wte(idx[None]) # token embeddings of shape (b, t, n_embd)
        x = norm(x) # @Grad62304977
        x0 = x
        vi = self.transformer.vte(idx[None]).chunk(self.num_encoder_layers, dim=-1)

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        for i in range(self.num_encoder_layers):
            x = self.transformer.h[i](x, vi[i], x0, block_mask)
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            # U-net structure on token value embeddings by @leloykun
            x = self.transformer.h[self.num_encoder_layers + i](x, vi[self.num_encoder_layers-1-i], x0, block_mask)

        x = norm(x)
        logits = self.lm_head(x)
        logits = self.lm_head_softcap * torch.tanh(logits / self.lm_head_softcap) # @Grad62304977
        logits = logits.float()
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), target.view(-1))
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _peek_data_shard(file: Path):
    # only reads the header, returns header data
    # header is 256 int32
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    return int(header[2]) # number of tokens (claimed)

def _load_data_shard(file: Path, ntok: int):
    with file.open("rb") as f:
        tokens = torch.empty(ntok, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy())
        assert nbytes == 2 * ntok, "number of tokens read does not match header?"
    return tokens

class DistributedDataLoader:
    def __init__(self, filename_pattern, T, process_rank, num_processes):
        self.process_rank = process_rank
        self.num_processes = num_processes
        self.T = T

        # glob files that match the pattern
        self.files = sorted(Path.cwd().glob(filename_pattern))
        assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"

        # load and validate all data shards, count number of tokens in total
        self.ntoks = [_peek_data_shard(file) for file in self.files]
        assert min(self.ntoks) >= num_processes * T + 1
        self.ntok_total = sum(self.ntoks)

        self.reset()

    def reset(self):
        self.current_shard = -1
        self.advance()

    def advance(self): # advance to next data shard
        self.current_shard = (self.current_shard + 1) % len(self.files)
        self.current_position = self.process_rank * self.T
        self.tokens = _load_data_shard(self.files[self.current_shard], self.ntoks[self.current_shard])

    def next_batch(self):
        batch_size = self.T * self.num_processes
        buf = self.tokens[self.current_position:self.current_position+self.T+1]
        # host side async is sufficient;
        # no performance improvement was observed when introducing a separate stream.
        x = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
        y = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
        # advance current position and load next shard if necessary
        self.current_position += batch_size
        if self.current_position + batch_size + 1 >= len(self.tokens):
            self.advance()
        return x, y

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data hyperparams
    input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
    input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
    # optimization hyperparams
    batch_size : int = 8 # batch size, in sequences, across all devices
    sequence_length : int = 64*1024 # sequence length, in tokens
    num_iterations : int = 1480 # number of iterations to run
    warmup_iters : int = 0
    cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
    weight_decay : float = 0
    # evaluation and logging hyperparams
    val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
    val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()

# set up DDP (distributed data parallel). torchrun sets this env variable
assert torch.cuda.is_available()
dist.init_process_group(backend='nccl')
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
device = f'cuda:{ddp_local_rank}'
torch.cuda.set_device(device)
print(f"using device: {device}")
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = str(uuid.uuid4())
    logdir = 'logs/%s/' % run_id
    # os.makedirs(logdir, exist_ok=True)
    logfile = 'logs/%s.txt' % run_id
    # create the log file
    with open(logfile, "w") as f:
        # begin the log by printing this file (the Python code)
        f.write(code)
        f.write('='*100 + '\n')
def print0(s, logonly=False):
    if master_process:
        with open(logfile, "a") as f:
            if not logonly:
                print(s)
            f.write(s+'\n')
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)

# convenience variables
T = args.sequence_length
# calculate the number of steps to take in the val loop.
assert args.val_tokens % (T * ddp_world_size) == 0
val_steps = args.val_tokens // (T * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size
assert train_accumulation_steps == 1

# load tokens
train_loader = DistributedDataLoader(args.input_bin, T, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, T, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.ntok_total} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.ntok_total} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
x, y = train_loader.next_batch()

# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, n_layer=12, n_head=6, n_embd=768))
model = model.cuda().bfloat16()
for m in model.modules():
    if isinstance(m, CastedLinear):
        m.float()
if hasattr(config, "coordinate_descent_tuning"):
    config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank])
raw_model = model.module # always contains the "raw" unwrapped model

# init the optimizer(s)
optimizer1 = torch.optim.Adam([raw_model.transformer.wte.weight, raw_model.transformer.vte.weight], lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.transformer.h.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
    assert it <= args.num_iterations
    # 1) linear warmup for warmup_iters steps
    if it < args.warmup_iters:
        return (it+1) / args.warmup_iters
    # 2) constant lr for a while
    elif it < args.num_iterations - args.cooldown_iters:
        return 1.0
    # 3) linear cooldown
    else:
        decay_ratio = (args.num_iterations - it) / args.cooldown_iters
        return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]

sliding_window_size = torch.tensor(64, dtype=torch.int32, device="cuda")
sw_size_prev = 64
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
    last_step = (step == args.num_iterations)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Set the sliding window size for the current step, in chunks of 64. By @fernbear.bsky.social
    sw_size =  64 * int((64 + (1792 - 64) * step / args.num_iterations) // 64)
    if sw_size != sw_size_prev:
        sliding_window_size.copy_(sw_size, non_blocking=True)
        sw_size_prev = sw_size

    # once in a while evaluate the validation dataset
    if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # run validation batches
        model.eval()
        val_loader.reset()
        val_loss = 0.0
        for _ in range(val_steps):
            with torch.no_grad():
                x_val, y_val = val_loader.next_batch()
                val_loss += model(x_val, y_val, sliding_window=sliding_window_size)
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        val_loss /= val_steps
        # log val loss to console and to logfile
        print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # save the state of the training process
        log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
        # torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    # bit confusing: we want to make sure to eval on 0th iteration
    # but also after the very last iteration. so we loop for step <= num_iterations
    # instead of just < num_iterations (one extra due to <=), only to do
    # the validation/sampling one last time, and then we break right here as we're done.
    if last_step:
        break

    # --------------- TRAINING SECTION BEGIN -----------------
    model.train()
    loss = model(x, y, sliding_window=sliding_window_size)
    loss.backward()
    del loss
    # advance the dataset for the next batch
    x, y = train_loader.next_batch()
    # momentum warmup for Muon
    frac = min(step/300, 1)
    for group in optimizer3.param_groups:
        group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # --------------- TRAINING SECTION END -------------------
    # everything that follows now is just diagnostics, prints, logging, etc.
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")

if master_process:
    print(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")

# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()
====================================================================================================
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Sun Dec  8 12:08:17 2024       
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.6     |
|-----------------------------------------+----------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
|                                         |                      |               MIG M. |
|=========================================+======================+======================|
|   0  NVIDIA H100 80GB HBM3          On  | 00000000:65:02.0 Off |                    0 |
| N/A   37C    P0              74W / 700W |      7MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  | 00000000:67:02.0 Off |                    0 |
| N/A   46C    P0             131W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  | 00000000:69:02.0 Off |                    0 |
| N/A   46C    P0             124W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  | 00000000:6B:02.0 Off |                    0 |
| N/A   40C    P0             119W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  | 00000000:6F:02.0 Off |                    0 |
| N/A   39C    P0             117W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  | 00000000:71:02.0 Off |                    0 |
| N/A   46C    P0             122W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  | 00000000:73:02.0 Off |                    0 |
| N/A   47C    P0             128W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  | 00000000:75:02.0 Off |                    0 |
| N/A   39C    P0             124W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
                                                                                         
+---------------------------------------------------------------------------------------+
| Processes:                                                                            |
|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
|        ID   ID                                                             Usage      |
|=======================================================================================|
+---------------------------------------------------------------------------------------+

====================================================================================================
Training DataLoader: total number of tokens: 3200000000 across 32 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:22921ms step_avg:nanms
step:2/1480 train_time:23009ms step_avg:nanms
step:3/1480 train_time:23147ms step_avg:nanms
step:4/1480 train_time:23290ms step_avg:nanms
step:5/1480 train_time:23432ms step_avg:nanms
step:6/1480 train_time:23574ms step_avg:nanms
step:7/1480 train_time:23716ms step_avg:nanms
step:8/1480 train_time:23857ms step_avg:nanms
step:9/1480 train_time:24002ms step_avg:nanms
step:10/1480 train_time:24146ms step_avg:nanms
step:11/1480 train_time:143ms step_avg:nanms
step:12/1480 train_time:286ms step_avg:nanms
step:13/1480 train_time:429ms step_avg:142.89ms
step:14/1480 train_time:570ms step_avg:142.38ms
step:15/1480 train_time:711ms step_avg:142.28ms
step:16/1480 train_time:855ms step_avg:142.51ms
step:17/1480 train_time:1000ms step_avg:142.80ms
step:18/1480 train_time:1144ms step_avg:143.05ms
step:19/1480 train_time:1288ms step_avg:143.10ms
step:20/1480 train_time:1430ms step_avg:143.03ms
step:21/1480 train_time:1571ms step_avg:142.86ms
step:22/1480 train_time:1713ms step_avg:142.71ms
step:23/1480 train_time:1855ms step_avg:142.69ms
step:24/1480 train_time:1999ms step_avg:142.82ms
step:25/1480 train_time:2144ms step_avg:142.94ms
step:26/1480 train_time:2288ms step_avg:143.02ms
step:27/1480 train_time:2431ms step_avg:143.00ms
step:28/1480 train_time:2572ms step_avg:142.89ms
step:29/1480 train_time:2712ms step_avg:142.76ms
step:30/1480 train_time:2854ms step_avg:142.70ms
step:31/1480 train_time:2996ms step_avg:142.67ms
step:32/1480 train_time:3140ms step_avg:142.73ms
step:33/1480 train_time:3283ms step_avg:142.75ms
step:34/1480 train_time:3427ms step_avg:142.78ms
step:35/1480 train_time:3568ms step_avg:142.74ms
step:36/1480 train_time:3710ms step_avg:142.69ms
step:37/1480 train_time:3853ms step_avg:142.69ms
step:38/1480 train_time:3995ms step_avg:142.67ms
step:39/1480 train_time:4139ms step_avg:142.73ms
step:40/1480 train_time:4283ms step_avg:142.77ms
step:41/1480 train_time:4428ms step_avg:142.83ms
step:42/1480 train_time:4570ms step_avg:142.83ms
step:43/1480 train_time:4712ms step_avg:142.78ms
step:44/1480 train_time:4854ms step_avg:142.77ms
step:45/1480 train_time:4997ms step_avg:142.76ms
step:46/1480 train_time:5138ms step_avg:142.72ms
step:47/1480 train_time:5280ms step_avg:142.69ms
step:48/1480 train_time:5424ms step_avg:142.74ms
step:49/1480 train_time:5567ms step_avg:142.76ms
step:50/1480 train_time:5709ms step_avg:142.72ms
step:51/1480 train_time:5851ms step_avg:142.71ms
step:52/1480 train_time:5993ms step_avg:142.69ms
step:53/1480 train_time:6135ms step_avg:142.67ms
step:54/1480 train_time:6276ms step_avg:142.65ms
step:55/1480 train_time:6418ms step_avg:142.62ms
step:56/1480 train_time:6562ms step_avg:142.65ms
step:57/1480 train_time:6705ms step_avg:142.65ms
step:58/1480 train_time:6847ms step_avg:142.65ms
step:59/1480 train_time:6988ms step_avg:142.62ms
step:60/1480 train_time:7130ms step_avg:142.59ms
step:61/1480 train_time:7272ms step_avg:142.59ms
step:62/1480 train_time:7414ms step_avg:142.57ms
step:63/1480 train_time:7557ms step_avg:142.59ms
step:64/1480 train_time:7702ms step_avg:142.63ms
step:65/1480 train_time:7845ms step_avg:142.64ms
step:66/1480 train_time:7987ms step_avg:142.63ms
step:67/1480 train_time:8129ms step_avg:142.62ms
step:68/1480 train_time:8270ms step_avg:142.59ms
step:69/1480 train_time:8412ms step_avg:142.57ms
step:70/1480 train_time:8555ms step_avg:142.58ms
step:71/1480 train_time:8699ms step_avg:142.61ms
step:72/1480 train_time:8842ms step_avg:142.61ms
step:73/1480 train_time:8985ms step_avg:142.62ms
step:74/1480 train_time:9127ms step_avg:142.62ms
step:75/1480 train_time:9269ms step_avg:142.60ms
step:76/1480 train_time:9411ms step_avg:142.59ms
step:77/1480 train_time:9553ms step_avg:142.58ms
step:78/1480 train_time:9695ms step_avg:142.57ms
step:79/1480 train_time:9837ms step_avg:142.57ms
step:80/1480 train_time:9982ms step_avg:142.60ms
step:81/1480 train_time:10125ms step_avg:142.61ms
step:82/1480 train_time:10268ms step_avg:142.61ms
step:83/1480 train_time:10409ms step_avg:142.59ms
step:84/1480 train_time:10551ms step_avg:142.58ms
step:85/1480 train_time:10694ms step_avg:142.59ms
step:86/1480 train_time:10836ms step_avg:142.57ms
step:87/1480 train_time:10977ms step_avg:142.56ms
step:88/1480 train_time:11121ms step_avg:142.58ms
step:89/1480 train_time:11264ms step_avg:142.58ms
step:90/1480 train_time:11406ms step_avg:142.58ms
step:91/1480 train_time:11549ms step_avg:142.58ms
step:92/1480 train_time:11691ms step_avg:142.57ms
step:93/1480 train_time:11833ms step_avg:142.57ms
step:94/1480 train_time:11974ms step_avg:142.55ms
step:95/1480 train_time:12117ms step_avg:142.56ms
step:96/1480 train_time:12260ms step_avg:142.56ms
step:97/1480 train_time:12404ms step_avg:142.58ms
step:98/1480 train_time:12547ms step_avg:142.58ms
step:99/1480 train_time:12689ms step_avg:142.58ms
step:100/1480 train_time:12832ms step_avg:142.58ms
step:101/1480 train_time:12973ms step_avg:142.56ms
step:102/1480 train_time:13114ms step_avg:142.55ms
step:103/1480 train_time:13257ms step_avg:142.54ms
step:104/1480 train_time:13400ms step_avg:142.55ms
step:105/1480 train_time:13544ms step_avg:142.57ms
step:106/1480 train_time:13688ms step_avg:142.58ms
step:107/1480 train_time:13830ms step_avg:142.58ms
step:108/1480 train_time:13971ms step_avg:142.57ms
step:109/1480 train_time:14114ms step_avg:142.56ms
step:110/1480 train_time:14257ms step_avg:142.57ms
step:111/1480 train_time:14403ms step_avg:142.61ms
step:112/1480 train_time:14551ms step_avg:142.65ms
step:113/1480 train_time:14697ms step_avg:142.69ms
step:114/1480 train_time:14845ms step_avg:142.74ms
step:115/1480 train_time:14992ms step_avg:142.78ms
step:116/1480 train_time:15138ms step_avg:142.81ms
step:117/1480 train_time:15285ms step_avg:142.85ms
step:118/1480 train_time:15432ms step_avg:142.89ms
step:119/1480 train_time:15579ms step_avg:142.92ms
step:120/1480 train_time:15726ms step_avg:142.97ms
step:121/1480 train_time:15874ms step_avg:143.01ms
step:122/1480 train_time:16019ms step_avg:143.03ms
step:123/1480 train_time:16167ms step_avg:143.07ms
step:124/1480 train_time:16313ms step_avg:143.10ms
step:125/1480 train_time:16460ms step_avg:143.13ms
step:125/1480 val_loss:4.4272 train_time:16517ms step_avg:143.62ms
step:126/1480 train_time:16612ms step_avg:143.21ms
step:127/1480 train_time:16762ms step_avg:143.27ms
step:128/1480 train_time:16909ms step_avg:143.29ms
step:129/1480 train_time:17054ms step_avg:143.31ms
step:130/1480 train_time:17200ms step_avg:143.33ms
step:131/1480 train_time:17346ms step_avg:143.35ms
step:132/1480 train_time:17492ms step_avg:143.38ms
step:133/1480 train_time:17640ms step_avg:143.41ms
step:134/1480 train_time:17789ms step_avg:143.46ms
step:135/1480 train_time:17934ms step_avg:143.48ms
step:136/1480 train_time:18083ms step_avg:143.51ms
step:137/1480 train_time:18229ms step_avg:143.53ms
step:138/1480 train_time:18374ms step_avg:143.54ms
step:139/1480 train_time:18520ms step_avg:143.57ms
step:140/1480 train_time:18669ms step_avg:143.61ms
step:141/1480 train_time:18814ms step_avg:143.62ms
step:142/1480 train_time:18962ms step_avg:143.65ms
step:143/1480 train_time:19109ms step_avg:143.68ms
step:144/1480 train_time:19255ms step_avg:143.69ms
step:145/1480 train_time:19401ms step_avg:143.71ms
step:146/1480 train_time:19549ms step_avg:143.74ms
step:147/1480 train_time:19696ms step_avg:143.76ms
step:148/1480 train_time:19843ms step_avg:143.79ms
step:149/1480 train_time:19990ms step_avg:143.82ms
step:150/1480 train_time:20136ms step_avg:143.83ms
step:151/1480 train_time:20284ms step_avg:143.86ms
step:152/1480 train_time:20430ms step_avg:143.87ms
step:153/1480 train_time:20576ms step_avg:143.88ms
step:154/1480 train_time:20722ms step_avg:143.90ms
step:155/1480 train_time:20870ms step_avg:143.93ms
step:156/1480 train_time:21015ms step_avg:143.94ms
step:157/1480 train_time:21163ms step_avg:143.96ms
step:158/1480 train_time:21309ms step_avg:143.98ms
step:159/1480 train_time:21454ms step_avg:143.99ms
step:160/1480 train_time:21603ms step_avg:144.02ms
step:161/1480 train_time:21750ms step_avg:144.04ms
step:162/1480 train_time:21896ms step_avg:144.05ms
step:163/1480 train_time:22043ms step_avg:144.07ms
step:164/1480 train_time:22190ms step_avg:144.09ms
step:165/1480 train_time:22335ms step_avg:144.09ms
step:166/1480 train_time:22483ms step_avg:144.12ms
step:167/1480 train_time:22630ms step_avg:144.14ms
step:168/1480 train_time:22776ms step_avg:144.15ms
step:169/1480 train_time:22923ms step_avg:144.17ms
step:170/1480 train_time:23069ms step_avg:144.18ms
step:171/1480 train_time:23215ms step_avg:144.19ms
step:172/1480 train_time:23361ms step_avg:144.20ms
step:173/1480 train_time:23508ms step_avg:144.22ms
step:174/1480 train_time:23654ms step_avg:144.23ms
step:175/1480 train_time:23800ms step_avg:144.25ms
step:176/1480 train_time:23948ms step_avg:144.26ms
step:177/1480 train_time:24095ms step_avg:144.28ms
step:178/1480 train_time:24242ms step_avg:144.30ms
step:179/1480 train_time:24390ms step_avg:144.32ms
step:180/1480 train_time:24535ms step_avg:144.32ms
step:181/1480 train_time:24683ms step_avg:144.34ms
step:182/1480 train_time:24829ms step_avg:144.36ms
step:183/1480 train_time:24976ms step_avg:144.37ms
step:184/1480 train_time:25124ms step_avg:144.39ms
step:185/1480 train_time:25271ms step_avg:144.41ms
step:186/1480 train_time:25417ms step_avg:144.41ms
step:187/1480 train_time:25565ms step_avg:144.44ms
step:188/1480 train_time:25712ms step_avg:144.45ms
step:189/1480 train_time:25858ms step_avg:144.46ms
step:190/1480 train_time:26005ms step_avg:144.47ms
step:191/1480 train_time:26151ms step_avg:144.48ms
step:192/1480 train_time:26298ms step_avg:144.49ms
step:193/1480 train_time:26445ms step_avg:144.51ms
step:194/1480 train_time:26592ms step_avg:144.52ms
step:195/1480 train_time:26738ms step_avg:144.53ms
step:196/1480 train_time:26886ms step_avg:144.55ms
step:197/1480 train_time:27032ms step_avg:144.55ms
step:198/1480 train_time:27178ms step_avg:144.56ms
step:199/1480 train_time:27325ms step_avg:144.57ms
step:200/1480 train_time:27472ms step_avg:144.59ms
step:201/1480 train_time:27618ms step_avg:144.59ms
step:202/1480 train_time:27766ms step_avg:144.62ms
step:203/1480 train_time:27913ms step_avg:144.62ms
step:204/1480 train_time:28059ms step_avg:144.64ms
step:205/1480 train_time:28207ms step_avg:144.65ms
step:206/1480 train_time:28351ms step_avg:144.65ms
step:207/1480 train_time:28498ms step_avg:144.66ms
step:208/1480 train_time:28644ms step_avg:144.67ms
step:209/1480 train_time:28792ms step_avg:144.69ms
step:210/1480 train_time:28940ms step_avg:144.70ms
step:211/1480 train_time:29087ms step_avg:144.71ms
step:212/1480 train_time:29232ms step_avg:144.71ms
step:213/1480 train_time:29379ms step_avg:144.72ms
step:214/1480 train_time:29527ms step_avg:144.74ms
step:215/1480 train_time:29673ms step_avg:144.75ms
step:216/1480 train_time:29819ms step_avg:144.75ms
step:217/1480 train_time:29967ms step_avg:144.77ms
step:218/1480 train_time:30113ms step_avg:144.77ms
step:219/1480 train_time:30259ms step_avg:144.78ms
step:220/1480 train_time:30406ms step_avg:144.79ms
step:221/1480 train_time:30553ms step_avg:144.80ms
step:222/1480 train_time:30705ms step_avg:144.83ms
step:223/1480 train_time:30854ms step_avg:144.86ms
step:224/1480 train_time:31005ms step_avg:144.88ms
step:225/1480 train_time:31155ms step_avg:144.91ms
step:226/1480 train_time:31306ms step_avg:144.94ms
step:227/1480 train_time:31457ms step_avg:144.96ms
step:228/1480 train_time:31607ms step_avg:144.99ms
step:229/1480 train_time:31758ms step_avg:145.02ms
step:230/1480 train_time:31908ms step_avg:145.04ms
step:231/1480 train_time:32057ms step_avg:145.05ms
step:232/1480 train_time:32207ms step_avg:145.08ms
step:233/1480 train_time:32359ms step_avg:145.11ms
step:234/1480 train_time:32511ms step_avg:145.14ms
step:235/1480 train_time:32662ms step_avg:145.17ms
step:236/1480 train_time:32813ms step_avg:145.19ms
step:237/1480 train_time:32963ms step_avg:145.21ms
step:238/1480 train_time:33113ms step_avg:145.23ms
step:239/1480 train_time:33264ms step_avg:145.26ms
step:240/1480 train_time:33414ms step_avg:145.28ms
step:241/1480 train_time:33567ms step_avg:145.31ms
step:242/1480 train_time:33716ms step_avg:145.33ms
step:243/1480 train_time:33867ms step_avg:145.35ms
step:244/1480 train_time:34016ms step_avg:145.37ms
step:245/1480 train_time:34166ms step_avg:145.39ms
step:246/1480 train_time:34316ms step_avg:145.41ms
step:247/1480 train_time:34467ms step_avg:145.43ms
step:248/1480 train_time:34618ms step_avg:145.45ms
step:249/1480 train_time:34768ms step_avg:145.47ms
step:250/1480 train_time:34919ms step_avg:145.50ms
step:250/1480 val_loss:3.9877 train_time:34977ms step_avg:145.74ms
step:251/1480 train_time:35072ms step_avg:145.53ms
step:252/1480 train_time:35225ms step_avg:145.56ms
step:253/1480 train_time:35373ms step_avg:145.57ms
step:254/1480 train_time:35524ms step_avg:145.59ms
step:255/1480 train_time:35672ms step_avg:145.60ms
step:256/1480 train_time:35822ms step_avg:145.62ms
step:257/1480 train_time:35972ms step_avg:145.64ms
step:258/1480 train_time:36124ms step_avg:145.66ms
step:259/1480 train_time:36276ms step_avg:145.69ms
step:260/1480 train_time:36427ms step_avg:145.71ms
step:261/1480 train_time:36576ms step_avg:145.72ms
step:262/1480 train_time:36726ms step_avg:145.74ms
step:263/1480 train_time:36874ms step_avg:145.75ms
step:264/1480 train_time:37026ms step_avg:145.77ms
step:265/1480 train_time:37175ms step_avg:145.78ms
step:266/1480 train_time:37327ms step_avg:145.81ms
step:267/1480 train_time:37477ms step_avg:145.83ms
step:268/1480 train_time:37628ms step_avg:145.84ms
step:269/1480 train_time:37777ms step_avg:145.86ms
step:270/1480 train_time:37927ms step_avg:145.87ms
step:271/1480 train_time:38077ms step_avg:145.89ms
step:272/1480 train_time:38227ms step_avg:145.91ms
step:273/1480 train_time:38377ms step_avg:145.92ms
step:274/1480 train_time:38528ms step_avg:145.94ms
step:275/1480 train_time:38679ms step_avg:145.96ms
step:276/1480 train_time:38829ms step_avg:145.97ms
step:277/1480 train_time:38978ms step_avg:145.98ms
step:278/1480 train_time:39129ms step_avg:146.00ms
step:279/1480 train_time:39279ms step_avg:146.02ms
step:280/1480 train_time:39429ms step_avg:146.03ms
step:281/1480 train_time:39580ms step_avg:146.05ms
step:282/1480 train_time:39730ms step_avg:146.07ms
step:283/1480 train_time:39881ms step_avg:146.08ms
step:284/1480 train_time:40031ms step_avg:146.10ms
step:285/1480 train_time:40182ms step_avg:146.12ms
step:286/1480 train_time:40332ms step_avg:146.13ms
step:287/1480 train_time:40484ms step_avg:146.15ms
step:288/1480 train_time:40634ms step_avg:146.16ms
step:289/1480 train_time:40785ms step_avg:146.18ms
step:290/1480 train_time:40935ms step_avg:146.20ms
step:291/1480 train_time:41086ms step_avg:146.21ms
step:292/1480 train_time:41235ms step_avg:146.22ms
step:293/1480 train_time:41387ms step_avg:146.24ms
step:294/1480 train_time:41537ms step_avg:146.26ms
step:295/1480 train_time:41688ms step_avg:146.27ms
step:296/1480 train_time:41838ms step_avg:146.29ms
step:297/1480 train_time:41990ms step_avg:146.31ms
step:298/1480 train_time:42140ms step_avg:146.32ms
step:299/1480 train_time:42291ms step_avg:146.34ms
step:300/1480 train_time:42441ms step_avg:146.35ms
step:301/1480 train_time:42591ms step_avg:146.36ms
step:302/1480 train_time:42741ms step_avg:146.37ms
step:303/1480 train_time:42892ms step_avg:146.39ms
step:304/1480 train_time:43043ms step_avg:146.40ms
step:305/1480 train_time:43193ms step_avg:146.42ms
step:306/1480 train_time:43343ms step_avg:146.43ms
step:307/1480 train_time:43494ms step_avg:146.45ms
step:308/1480 train_time:43645ms step_avg:146.46ms
step:309/1480 train_time:43797ms step_avg:146.48ms
step:310/1480 train_time:43948ms step_avg:146.49ms
step:311/1480 train_time:44098ms step_avg:146.51ms
step:312/1480 train_time:44249ms step_avg:146.52ms
step:313/1480 train_time:44399ms step_avg:146.53ms
step:314/1480 train_time:44549ms step_avg:146.54ms
step:315/1480 train_time:44700ms step_avg:146.56ms
step:316/1480 train_time:44849ms step_avg:146.57ms
step:317/1480 train_time:45001ms step_avg:146.58ms
step:318/1480 train_time:45151ms step_avg:146.59ms
step:319/1480 train_time:45302ms step_avg:146.61ms
step:320/1480 train_time:45452ms step_avg:146.62ms
step:321/1480 train_time:45603ms step_avg:146.63ms
step:322/1480 train_time:45753ms step_avg:146.64ms
step:323/1480 train_time:45904ms step_avg:146.66ms
step:324/1480 train_time:46053ms step_avg:146.67ms
step:325/1480 train_time:46203ms step_avg:146.67ms
step:326/1480 train_time:46353ms step_avg:146.69ms
step:327/1480 train_time:46504ms step_avg:146.70ms
step:328/1480 train_time:46653ms step_avg:146.71ms
step:329/1480 train_time:46804ms step_avg:146.72ms
step:330/1480 train_time:46956ms step_avg:146.74ms
step:331/1480 train_time:47109ms step_avg:146.76ms
step:332/1480 train_time:47262ms step_avg:146.78ms
step:333/1480 train_time:47416ms step_avg:146.80ms
step:334/1480 train_time:47568ms step_avg:146.82ms
step:335/1480 train_time:47722ms step_avg:146.84ms
step:336/1480 train_time:47875ms step_avg:146.85ms
step:337/1480 train_time:48028ms step_avg:146.87ms
step:338/1480 train_time:48182ms step_avg:146.90ms
step:339/1480 train_time:48334ms step_avg:146.91ms
step:340/1480 train_time:48490ms step_avg:146.94ms
step:341/1480 train_time:48644ms step_avg:146.96ms
step:342/1480 train_time:48798ms step_avg:146.98ms
step:343/1480 train_time:48952ms step_avg:147.00ms
step:344/1480 train_time:49106ms step_avg:147.02ms
step:345/1480 train_time:49262ms step_avg:147.05ms
step:346/1480 train_time:49416ms step_avg:147.07ms
step:347/1480 train_time:49570ms step_avg:147.09ms
step:348/1480 train_time:49725ms step_avg:147.12ms
step:349/1480 train_time:49879ms step_avg:147.13ms
step:350/1480 train_time:50032ms step_avg:147.15ms
step:351/1480 train_time:50186ms step_avg:147.17ms
step:352/1480 train_time:50340ms step_avg:147.19ms
step:353/1480 train_time:50495ms step_avg:147.22ms
step:354/1480 train_time:50649ms step_avg:147.23ms
step:355/1480 train_time:50804ms step_avg:147.26ms
step:356/1480 train_time:50957ms step_avg:147.27ms
step:357/1480 train_time:51110ms step_avg:147.29ms
step:358/1480 train_time:51263ms step_avg:147.31ms
step:359/1480 train_time:51418ms step_avg:147.33ms
step:360/1480 train_time:51572ms step_avg:147.35ms
step:361/1480 train_time:51727ms step_avg:147.37ms
step:362/1480 train_time:51882ms step_avg:147.39ms
step:363/1480 train_time:52035ms step_avg:147.41ms
step:364/1480 train_time:52190ms step_avg:147.43ms
step:365/1480 train_time:52343ms step_avg:147.44ms
step:366/1480 train_time:52496ms step_avg:147.46ms
step:367/1480 train_time:52650ms step_avg:147.48ms
step:368/1480 train_time:52804ms step_avg:147.50ms
step:369/1480 train_time:52957ms step_avg:147.51ms
step:370/1480 train_time:53110ms step_avg:147.53ms
step:371/1480 train_time:53263ms step_avg:147.54ms
step:372/1480 train_time:53418ms step_avg:147.56ms
step:373/1480 train_time:53571ms step_avg:147.58ms
step:374/1480 train_time:53725ms step_avg:147.60ms
step:375/1480 train_time:53880ms step_avg:147.62ms
step:375/1480 val_loss:3.8043 train_time:53941ms step_avg:147.78ms
step:376/1480 train_time:54039ms step_avg:147.65ms
step:377/1480 train_time:54194ms step_avg:147.67ms
step:378/1480 train_time:54347ms step_avg:147.68ms
step:379/1480 train_time:54501ms step_avg:147.70ms
step:380/1480 train_time:54653ms step_avg:147.71ms
step:381/1480 train_time:54805ms step_avg:147.72ms
step:382/1480 train_time:54958ms step_avg:147.74ms
step:383/1480 train_time:55114ms step_avg:147.76ms
step:384/1480 train_time:55268ms step_avg:147.78ms
step:385/1480 train_time:55423ms step_avg:147.80ms
step:386/1480 train_time:55576ms step_avg:147.81ms
step:387/1480 train_time:55729ms step_avg:147.82ms
step:388/1480 train_time:55882ms step_avg:147.84ms
step:389/1480 train_time:56035ms step_avg:147.85ms
step:390/1480 train_time:56191ms step_avg:147.87ms
step:391/1480 train_time:56346ms step_avg:147.89ms
step:392/1480 train_time:56500ms step_avg:147.90ms
step:393/1480 train_time:56652ms step_avg:147.92ms
step:394/1480 train_time:56806ms step_avg:147.93ms
step:395/1480 train_time:56959ms step_avg:147.95ms
step:396/1480 train_time:57112ms step_avg:147.96ms
step:397/1480 train_time:57266ms step_avg:147.97ms
step:398/1480 train_time:57422ms step_avg:147.99ms
step:399/1480 train_time:57576ms step_avg:148.01ms
step:400/1480 train_time:57731ms step_avg:148.03ms
step:401/1480 train_time:57884ms step_avg:148.04ms
step:402/1480 train_time:58037ms step_avg:148.05ms
step:403/1480 train_time:58191ms step_avg:148.07ms
step:404/1480 train_time:58345ms step_avg:148.08ms
step:405/1480 train_time:58502ms step_avg:148.11ms
step:406/1480 train_time:58655ms step_avg:148.12ms
step:407/1480 train_time:58809ms step_avg:148.13ms
step:408/1480 train_time:58963ms step_avg:148.15ms
step:409/1480 train_time:59117ms step_avg:148.16ms
step:410/1480 train_time:59268ms step_avg:148.17ms
step:411/1480 train_time:59422ms step_avg:148.19ms
step:412/1480 train_time:59576ms step_avg:148.20ms
step:413/1480 train_time:59729ms step_avg:148.21ms
step:414/1480 train_time:59884ms step_avg:148.23ms
step:415/1480 train_time:60037ms step_avg:148.24ms
step:416/1480 train_time:60191ms step_avg:148.25ms
step:417/1480 train_time:60344ms step_avg:148.27ms
step:418/1480 train_time:60498ms step_avg:148.28ms
step:419/1480 train_time:60651ms step_avg:148.29ms
step:420/1480 train_time:60805ms step_avg:148.30ms
step:421/1480 train_time:60958ms step_avg:148.32ms
step:422/1480 train_time:61111ms step_avg:148.33ms
step:423/1480 train_time:61264ms step_avg:148.34ms
step:424/1480 train_time:61419ms step_avg:148.35ms
step:425/1480 train_time:61572ms step_avg:148.37ms
step:426/1480 train_time:61726ms step_avg:148.38ms
step:427/1480 train_time:61879ms step_avg:148.39ms
step:428/1480 train_time:62032ms step_avg:148.40ms
step:429/1480 train_time:62186ms step_avg:148.41ms
step:430/1480 train_time:62340ms step_avg:148.43ms
step:431/1480 train_time:62493ms step_avg:148.44ms
step:432/1480 train_time:62647ms step_avg:148.45ms
step:433/1480 train_time:62803ms step_avg:148.47ms
step:434/1480 train_time:62957ms step_avg:148.48ms
step:435/1480 train_time:63110ms step_avg:148.49ms
step:436/1480 train_time:63264ms step_avg:148.51ms
step:437/1480 train_time:63416ms step_avg:148.52ms
step:438/1480 train_time:63569ms step_avg:148.53ms
step:439/1480 train_time:63724ms step_avg:148.54ms
step:440/1480 train_time:63880ms step_avg:148.56ms
step:441/1480 train_time:64036ms step_avg:148.58ms
step:442/1480 train_time:64193ms step_avg:148.59ms
step:443/1480 train_time:64350ms step_avg:148.61ms
step:444/1480 train_time:64506ms step_avg:148.63ms
step:445/1480 train_time:64663ms step_avg:148.65ms
step:446/1480 train_time:64820ms step_avg:148.67ms
step:447/1480 train_time:64977ms step_avg:148.69ms
step:448/1480 train_time:65133ms step_avg:148.70ms
step:449/1480 train_time:65291ms step_avg:148.73ms
step:450/1480 train_time:65447ms step_avg:148.74ms
step:451/1480 train_time:65606ms step_avg:148.77ms
step:452/1480 train_time:65763ms step_avg:148.79ms
step:453/1480 train_time:65921ms step_avg:148.81ms
step:454/1480 train_time:66078ms step_avg:148.82ms
step:455/1480 train_time:66233ms step_avg:148.84ms
step:456/1480 train_time:66391ms step_avg:148.86ms
step:457/1480 train_time:66546ms step_avg:148.87ms
step:458/1480 train_time:66704ms step_avg:148.89ms
step:459/1480 train_time:66861ms step_avg:148.91ms
step:460/1480 train_time:67019ms step_avg:148.93ms
step:461/1480 train_time:67176ms step_avg:148.95ms
step:462/1480 train_time:67331ms step_avg:148.96ms
step:463/1480 train_time:67488ms step_avg:148.98ms
step:464/1480 train_time:67645ms step_avg:149.00ms
step:465/1480 train_time:67800ms step_avg:149.01ms
step:466/1480 train_time:67956ms step_avg:149.03ms
step:467/1480 train_time:68113ms step_avg:149.04ms
step:468/1480 train_time:68270ms step_avg:149.06ms
step:469/1480 train_time:68428ms step_avg:149.08ms
step:470/1480 train_time:68586ms step_avg:149.10ms
step:471/1480 train_time:68744ms step_avg:149.12ms
step:472/1480 train_time:68902ms step_avg:149.14ms
step:473/1480 train_time:69059ms step_avg:149.16ms
step:474/1480 train_time:69215ms step_avg:149.17ms
step:475/1480 train_time:69371ms step_avg:149.18ms
step:476/1480 train_time:69528ms step_avg:149.20ms
step:477/1480 train_time:69686ms step_avg:149.22ms
step:478/1480 train_time:69843ms step_avg:149.24ms
step:479/1480 train_time:70001ms step_avg:149.26ms
step:480/1480 train_time:70159ms step_avg:149.27ms
step:481/1480 train_time:70316ms step_avg:149.29ms
step:482/1480 train_time:70472ms step_avg:149.31ms
step:483/1480 train_time:70629ms step_avg:149.32ms
step:484/1480 train_time:70785ms step_avg:149.33ms
step:485/1480 train_time:70943ms step_avg:149.35ms
step:486/1480 train_time:71100ms step_avg:149.37ms
step:487/1480 train_time:71257ms step_avg:149.38ms
step:488/1480 train_time:71413ms step_avg:149.40ms
step:489/1480 train_time:71569ms step_avg:149.41ms
step:490/1480 train_time:71727ms step_avg:149.43ms
step:491/1480 train_time:71884ms step_avg:149.45ms
step:492/1480 train_time:72042ms step_avg:149.46ms
step:493/1480 train_time:72199ms step_avg:149.48ms
step:494/1480 train_time:72355ms step_avg:149.49ms
step:495/1480 train_time:72513ms step_avg:149.51ms
step:496/1480 train_time:72670ms step_avg:149.53ms
step:497/1480 train_time:72827ms step_avg:149.54ms
step:498/1480 train_time:72985ms step_avg:149.56ms
step:499/1480 train_time:73144ms step_avg:149.58ms
step:500/1480 train_time:73301ms step_avg:149.59ms
step:500/1480 val_loss:3.6840 train_time:73362ms step_avg:149.72ms
step:501/1480 train_time:73460ms step_avg:149.61ms
step:502/1480 train_time:73618ms step_avg:149.63ms
step:503/1480 train_time:73774ms step_avg:149.64ms
step:504/1480 train_time:73930ms step_avg:149.66ms
step:505/1480 train_time:74085ms step_avg:149.67ms
step:506/1480 train_time:74241ms step_avg:149.68ms
step:507/1480 train_time:74397ms step_avg:149.69ms
step:508/1480 train_time:74554ms step_avg:149.71ms
step:509/1480 train_time:74711ms step_avg:149.72ms
step:510/1480 train_time:74868ms step_avg:149.74ms
step:511/1480 train_time:75025ms step_avg:149.75ms
step:512/1480 train_time:75182ms step_avg:149.76ms
step:513/1480 train_time:75338ms step_avg:149.78ms
step:514/1480 train_time:75495ms step_avg:149.79ms
step:515/1480 train_time:75652ms step_avg:149.81ms
step:516/1480 train_time:75813ms step_avg:149.83ms
step:517/1480 train_time:75971ms step_avg:149.84ms
step:518/1480 train_time:76129ms step_avg:149.86ms
step:519/1480 train_time:76287ms step_avg:149.88ms
step:520/1480 train_time:76446ms step_avg:149.89ms
step:521/1480 train_time:76603ms step_avg:149.91ms
step:522/1480 train_time:76760ms step_avg:149.92ms
step:523/1480 train_time:76916ms step_avg:149.93ms
step:524/1480 train_time:77074ms step_avg:149.95ms
step:525/1480 train_time:77232ms step_avg:149.96ms
step:526/1480 train_time:77391ms step_avg:149.98ms
step:527/1480 train_time:77547ms step_avg:149.99ms
step:528/1480 train_time:77702ms step_avg:150.00ms
step:529/1480 train_time:77858ms step_avg:150.02ms
step:530/1480 train_time:78015ms step_avg:150.03ms
step:531/1480 train_time:78172ms step_avg:150.04ms
step:532/1480 train_time:78329ms step_avg:150.05ms
step:533/1480 train_time:78487ms step_avg:150.07ms
step:534/1480 train_time:78643ms step_avg:150.08ms
step:535/1480 train_time:78799ms step_avg:150.09ms
step:536/1480 train_time:78956ms step_avg:150.11ms
step:537/1480 train_time:79115ms step_avg:150.12ms
step:538/1480 train_time:79273ms step_avg:150.14ms
step:539/1480 train_time:79432ms step_avg:150.15ms
step:540/1480 train_time:79590ms step_avg:150.17ms
step:541/1480 train_time:79747ms step_avg:150.18ms
step:542/1480 train_time:79903ms step_avg:150.19ms
step:543/1480 train_time:80059ms step_avg:150.20ms
step:544/1480 train_time:80215ms step_avg:150.21ms
step:545/1480 train_time:80371ms step_avg:150.23ms
step:546/1480 train_time:80529ms step_avg:150.24ms
step:547/1480 train_time:80687ms step_avg:150.26ms
step:548/1480 train_time:80845ms step_avg:150.27ms
step:549/1480 train_time:81001ms step_avg:150.28ms
step:550/1480 train_time:81158ms step_avg:150.29ms
step:551/1480 train_time:81316ms step_avg:150.31ms
step:552/1480 train_time:81475ms step_avg:150.32ms
step:553/1480 train_time:81635ms step_avg:150.34ms
step:554/1480 train_time:81794ms step_avg:150.36ms
step:555/1480 train_time:81954ms step_avg:150.37ms
step:556/1480 train_time:82113ms step_avg:150.39ms
step:557/1480 train_time:82274ms step_avg:150.41ms
step:558/1480 train_time:82434ms step_avg:150.43ms
step:559/1480 train_time:82594ms step_avg:150.44ms
step:560/1480 train_time:82753ms step_avg:150.46ms
step:561/1480 train_time:82912ms step_avg:150.48ms
step:562/1480 train_time:83073ms step_avg:150.49ms
step:563/1480 train_time:83232ms step_avg:150.51ms
step:564/1480 train_time:83393ms step_avg:150.53ms
step:565/1480 train_time:83553ms step_avg:150.55ms
step:566/1480 train_time:83714ms step_avg:150.56ms
step:567/1480 train_time:83873ms step_avg:150.58ms
step:568/1480 train_time:84031ms step_avg:150.59ms
step:569/1480 train_time:84192ms step_avg:150.61ms
step:570/1480 train_time:84351ms step_avg:150.63ms
step:571/1480 train_time:84512ms step_avg:150.65ms
step:572/1480 train_time:84672ms step_avg:150.66ms
step:573/1480 train_time:84834ms step_avg:150.68ms
step:574/1480 train_time:84995ms step_avg:150.70ms
step:575/1480 train_time:85155ms step_avg:150.72ms
step:576/1480 train_time:85315ms step_avg:150.73ms
step:577/1480 train_time:85475ms step_avg:150.75ms
step:578/1480 train_time:85634ms step_avg:150.76ms
step:579/1480 train_time:85794ms step_avg:150.78ms
step:580/1480 train_time:85953ms step_avg:150.79ms
step:581/1480 train_time:86113ms step_avg:150.81ms
step:582/1480 train_time:86273ms step_avg:150.83ms
step:583/1480 train_time:86432ms step_avg:150.84ms
step:584/1480 train_time:86593ms step_avg:150.86ms
step:585/1480 train_time:86752ms step_avg:150.87ms
step:586/1480 train_time:86913ms step_avg:150.89ms
step:587/1480 train_time:87073ms step_avg:150.91ms
step:588/1480 train_time:87232ms step_avg:150.92ms
step:589/1480 train_time:87393ms step_avg:150.94ms
step:590/1480 train_time:87554ms step_avg:150.95ms
step:591/1480 train_time:87713ms step_avg:150.97ms
step:592/1480 train_time:87872ms step_avg:150.98ms
step:593/1480 train_time:88034ms step_avg:151.00ms
step:594/1480 train_time:88195ms step_avg:151.02ms
step:595/1480 train_time:88355ms step_avg:151.03ms
step:596/1480 train_time:88516ms step_avg:151.05ms
step:597/1480 train_time:88675ms step_avg:151.06ms
step:598/1480 train_time:88832ms step_avg:151.07ms
step:599/1480 train_time:88991ms step_avg:151.09ms
step:600/1480 train_time:89150ms step_avg:151.10ms
step:601/1480 train_time:89310ms step_avg:151.12ms
step:602/1480 train_time:89471ms step_avg:151.13ms
step:603/1480 train_time:89632ms step_avg:151.15ms
step:604/1480 train_time:89792ms step_avg:151.17ms
step:605/1480 train_time:89951ms step_avg:151.18ms
step:606/1480 train_time:90114ms step_avg:151.20ms
step:607/1480 train_time:90276ms step_avg:151.22ms
step:608/1480 train_time:90435ms step_avg:151.23ms
step:609/1480 train_time:90594ms step_avg:151.24ms
step:610/1480 train_time:90752ms step_avg:151.25ms
step:611/1480 train_time:90913ms step_avg:151.27ms
step:612/1480 train_time:91074ms step_avg:151.29ms
step:613/1480 train_time:91235ms step_avg:151.30ms
step:614/1480 train_time:91395ms step_avg:151.32ms
step:615/1480 train_time:91553ms step_avg:151.33ms
step:616/1480 train_time:91714ms step_avg:151.34ms
step:617/1480 train_time:91873ms step_avg:151.36ms
step:618/1480 train_time:92032ms step_avg:151.37ms
step:619/1480 train_time:92193ms step_avg:151.38ms
step:620/1480 train_time:92352ms step_avg:151.40ms
step:621/1480 train_time:92514ms step_avg:151.41ms
step:622/1480 train_time:92674ms step_avg:151.43ms
step:623/1480 train_time:92834ms step_avg:151.44ms
step:624/1480 train_time:92993ms step_avg:151.45ms
step:625/1480 train_time:93153ms step_avg:151.47ms
step:625/1480 val_loss:3.6035 train_time:93217ms step_avg:151.57ms
step:626/1480 train_time:93316ms step_avg:151.49ms
step:627/1480 train_time:93475ms step_avg:151.50ms
step:628/1480 train_time:93633ms step_avg:151.51ms
step:629/1480 train_time:93791ms step_avg:151.52ms
step:630/1480 train_time:93949ms step_avg:151.53ms
step:631/1480 train_time:94106ms step_avg:151.54ms
step:632/1480 train_time:94265ms step_avg:151.55ms
step:633/1480 train_time:94424ms step_avg:151.56ms
step:634/1480 train_time:94586ms step_avg:151.58ms
step:635/1480 train_time:94745ms step_avg:151.59ms
step:636/1480 train_time:94905ms step_avg:151.60ms
step:637/1480 train_time:95065ms step_avg:151.62ms
step:638/1480 train_time:95225ms step_avg:151.63ms
step:639/1480 train_time:95385ms step_avg:151.65ms
step:640/1480 train_time:95546ms step_avg:151.66ms
step:641/1480 train_time:95706ms step_avg:151.67ms
step:642/1480 train_time:95865ms step_avg:151.69ms
step:643/1480 train_time:96024ms step_avg:151.70ms
step:644/1480 train_time:96183ms step_avg:151.71ms
step:645/1480 train_time:96342ms step_avg:151.72ms
step:646/1480 train_time:96502ms step_avg:151.73ms
step:647/1480 train_time:96660ms step_avg:151.74ms
step:648/1480 train_time:96822ms step_avg:151.76ms
step:649/1480 train_time:96983ms step_avg:151.77ms
step:650/1480 train_time:97143ms step_avg:151.79ms
step:651/1480 train_time:97303ms step_avg:151.80ms
step:652/1480 train_time:97464ms step_avg:151.81ms
step:653/1480 train_time:97623ms step_avg:151.82ms
step:654/1480 train_time:97784ms step_avg:151.84ms
step:655/1480 train_time:97943ms step_avg:151.85ms
step:656/1480 train_time:98103ms step_avg:151.86ms
step:657/1480 train_time:98264ms step_avg:151.88ms
step:658/1480 train_time:98425ms step_avg:151.89ms
step:659/1480 train_time:98587ms step_avg:151.91ms
step:660/1480 train_time:98748ms step_avg:151.92ms
step:661/1480 train_time:98909ms step_avg:151.93ms
step:662/1480 train_time:99069ms step_avg:151.95ms
step:663/1480 train_time:99227ms step_avg:151.96ms
step:664/1480 train_time:99390ms step_avg:151.97ms
step:665/1480 train_time:99552ms step_avg:151.99ms
step:666/1480 train_time:99712ms step_avg:152.00ms
step:667/1480 train_time:99873ms step_avg:152.01ms
step:668/1480 train_time:100034ms step_avg:152.03ms
step:669/1480 train_time:100196ms step_avg:152.04ms
step:670/1480 train_time:100357ms step_avg:152.06ms
step:671/1480 train_time:100519ms step_avg:152.07ms
step:672/1480 train_time:100682ms step_avg:152.09ms
step:673/1480 train_time:100846ms step_avg:152.11ms
step:674/1480 train_time:101009ms step_avg:152.12ms
step:675/1480 train_time:101171ms step_avg:152.14ms
step:676/1480 train_time:101332ms step_avg:152.15ms
step:677/1480 train_time:101491ms step_avg:152.16ms
step:678/1480 train_time:101651ms step_avg:152.17ms
step:679/1480 train_time:101812ms step_avg:152.19ms
step:680/1480 train_time:101975ms step_avg:152.20ms
step:681/1480 train_time:102134ms step_avg:152.21ms
step:682/1480 train_time:102296ms step_avg:152.23ms
step:683/1480 train_time:102457ms step_avg:152.24ms
step:684/1480 train_time:102617ms step_avg:152.25ms
step:685/1480 train_time:102781ms step_avg:152.27ms
step:686/1480 train_time:102944ms step_avg:152.28ms
step:687/1480 train_time:103105ms step_avg:152.30ms
step:688/1480 train_time:103270ms step_avg:152.32ms
step:689/1480 train_time:103433ms step_avg:152.33ms
step:690/1480 train_time:103595ms step_avg:152.34ms
step:691/1480 train_time:103754ms step_avg:152.35ms
step:692/1480 train_time:103913ms step_avg:152.36ms
step:693/1480 train_time:104073ms step_avg:152.38ms
step:694/1480 train_time:104233ms step_avg:152.39ms
step:695/1480 train_time:104393ms step_avg:152.40ms
step:696/1480 train_time:104553ms step_avg:152.41ms
step:697/1480 train_time:104715ms step_avg:152.42ms
step:698/1480 train_time:104876ms step_avg:152.44ms
step:699/1480 train_time:105039ms step_avg:152.45ms
step:700/1480 train_time:105202ms step_avg:152.47ms
step:701/1480 train_time:105363ms step_avg:152.48ms
step:702/1480 train_time:105525ms step_avg:152.49ms
step:703/1480 train_time:105687ms step_avg:152.51ms
step:704/1480 train_time:105847ms step_avg:152.52ms
step:705/1480 train_time:106010ms step_avg:152.53ms
step:706/1480 train_time:106174ms step_avg:152.55ms
step:707/1480 train_time:106335ms step_avg:152.56ms
step:708/1480 train_time:106496ms step_avg:152.57ms
step:709/1480 train_time:106657ms step_avg:152.58ms
step:710/1480 train_time:106817ms step_avg:152.60ms
step:711/1480 train_time:106981ms step_avg:152.61ms
step:712/1480 train_time:107148ms step_avg:152.63ms
step:713/1480 train_time:107310ms step_avg:152.65ms
step:714/1480 train_time:107470ms step_avg:152.66ms
step:715/1480 train_time:107629ms step_avg:152.67ms
step:716/1480 train_time:107789ms step_avg:152.68ms
step:717/1480 train_time:107952ms step_avg:152.69ms
step:718/1480 train_time:108111ms step_avg:152.70ms
step:719/1480 train_time:108271ms step_avg:152.71ms
step:720/1480 train_time:108434ms step_avg:152.72ms
step:721/1480 train_time:108595ms step_avg:152.74ms
step:722/1480 train_time:108756ms step_avg:152.75ms
step:723/1480 train_time:108915ms step_avg:152.76ms
step:724/1480 train_time:109077ms step_avg:152.77ms
step:725/1480 train_time:109241ms step_avg:152.78ms
step:726/1480 train_time:109406ms step_avg:152.80ms
step:727/1480 train_time:109569ms step_avg:152.82ms
step:728/1480 train_time:109728ms step_avg:152.83ms
step:729/1480 train_time:109890ms step_avg:152.84ms
step:730/1480 train_time:110054ms step_avg:152.85ms
step:731/1480 train_time:110215ms step_avg:152.86ms
step:732/1480 train_time:110373ms step_avg:152.87ms
step:733/1480 train_time:110534ms step_avg:152.88ms
step:734/1480 train_time:110696ms step_avg:152.89ms
step:735/1480 train_time:110856ms step_avg:152.91ms
step:736/1480 train_time:111018ms step_avg:152.92ms
step:737/1480 train_time:111180ms step_avg:152.93ms
step:738/1480 train_time:111341ms step_avg:152.94ms
step:739/1480 train_time:111502ms step_avg:152.95ms
step:740/1480 train_time:111669ms step_avg:152.97ms
step:741/1480 train_time:111832ms step_avg:152.99ms
step:742/1480 train_time:111993ms step_avg:153.00ms
step:743/1480 train_time:112154ms step_avg:153.01ms
step:744/1480 train_time:112318ms step_avg:153.02ms
step:745/1480 train_time:112485ms step_avg:153.04ms
step:746/1480 train_time:112645ms step_avg:153.05ms
step:747/1480 train_time:112806ms step_avg:153.06ms
step:748/1480 train_time:112971ms step_avg:153.08ms
step:749/1480 train_time:113134ms step_avg:153.09ms
step:750/1480 train_time:113292ms step_avg:153.10ms
step:750/1480 val_loss:3.5487 train_time:113357ms step_avg:153.18ms
step:751/1480 train_time:113458ms step_avg:153.11ms
step:752/1480 train_time:113624ms step_avg:153.13ms
step:753/1480 train_time:113785ms step_avg:153.14ms
step:754/1480 train_time:113946ms step_avg:153.15ms
step:755/1480 train_time:114108ms step_avg:153.16ms
step:756/1480 train_time:114269ms step_avg:153.18ms
step:757/1480 train_time:114435ms step_avg:153.19ms
step:758/1480 train_time:114593ms step_avg:153.20ms
step:759/1480 train_time:114756ms step_avg:153.21ms
step:760/1480 train_time:114916ms step_avg:153.22ms
step:761/1480 train_time:115079ms step_avg:153.23ms
step:762/1480 train_time:115240ms step_avg:153.25ms
step:763/1480 train_time:115403ms step_avg:153.26ms
step:764/1480 train_time:115566ms step_avg:153.27ms
step:765/1480 train_time:115729ms step_avg:153.28ms
step:766/1480 train_time:115891ms step_avg:153.29ms
step:767/1480 train_time:116052ms step_avg:153.31ms
step:768/1480 train_time:116213ms step_avg:153.32ms
step:769/1480 train_time:116375ms step_avg:153.33ms
step:770/1480 train_time:116538ms step_avg:153.34ms
step:771/1480 train_time:116703ms step_avg:153.35ms
step:772/1480 train_time:116866ms step_avg:153.37ms
step:773/1480 train_time:117029ms step_avg:153.38ms
step:774/1480 train_time:117191ms step_avg:153.39ms
step:775/1480 train_time:117353ms step_avg:153.40ms
step:776/1480 train_time:117516ms step_avg:153.41ms
step:777/1480 train_time:117681ms step_avg:153.43ms
step:778/1480 train_time:117845ms step_avg:153.44ms
step:779/1480 train_time:118008ms step_avg:153.46ms
step:780/1480 train_time:118172ms step_avg:153.47ms
step:781/1480 train_time:118336ms step_avg:153.48ms
step:782/1480 train_time:118499ms step_avg:153.50ms
step:783/1480 train_time:118660ms step_avg:153.51ms
step:784/1480 train_time:118825ms step_avg:153.52ms
step:785/1480 train_time:118988ms step_avg:153.53ms
step:786/1480 train_time:119153ms step_avg:153.55ms
step:787/1480 train_time:119316ms step_avg:153.56ms
step:788/1480 train_time:119482ms step_avg:153.58ms
step:789/1480 train_time:119645ms step_avg:153.59ms
step:790/1480 train_time:119809ms step_avg:153.60ms
step:791/1480 train_time:119975ms step_avg:153.62ms
step:792/1480 train_time:120139ms step_avg:153.63ms
step:793/1480 train_time:120302ms step_avg:153.64ms
step:794/1480 train_time:120467ms step_avg:153.66ms
step:795/1480 train_time:120632ms step_avg:153.67ms
step:796/1480 train_time:120798ms step_avg:153.69ms
step:797/1480 train_time:120962ms step_avg:153.70ms
step:798/1480 train_time:121127ms step_avg:153.71ms
step:799/1480 train_time:121293ms step_avg:153.73ms
step:800/1480 train_time:121455ms step_avg:153.74ms
step:801/1480 train_time:121618ms step_avg:153.75ms
step:802/1480 train_time:121787ms step_avg:153.77ms
step:803/1480 train_time:121950ms step_avg:153.78ms
step:804/1480 train_time:122113ms step_avg:153.79ms
step:805/1480 train_time:122276ms step_avg:153.81ms
step:806/1480 train_time:122439ms step_avg:153.82ms
step:807/1480 train_time:122600ms step_avg:153.83ms
step:808/1480 train_time:122766ms step_avg:153.84ms
step:809/1480 train_time:122930ms step_avg:153.85ms
step:810/1480 train_time:123092ms step_avg:153.86ms
step:811/1480 train_time:123253ms step_avg:153.87ms
step:812/1480 train_time:123418ms step_avg:153.89ms
step:813/1480 train_time:123581ms step_avg:153.90ms
step:814/1480 train_time:123745ms step_avg:153.91ms
step:815/1480 train_time:123907ms step_avg:153.92ms
step:816/1480 train_time:124072ms step_avg:153.94ms
step:817/1480 train_time:124234ms step_avg:153.95ms
step:818/1480 train_time:124394ms step_avg:153.95ms
step:819/1480 train_time:124558ms step_avg:153.97ms
step:820/1480 train_time:124723ms step_avg:153.98ms
step:821/1480 train_time:124886ms step_avg:153.99ms
step:822/1480 train_time:125049ms step_avg:154.00ms
step:823/1480 train_time:125211ms step_avg:154.01ms
step:824/1480 train_time:125373ms step_avg:154.02ms
step:825/1480 train_time:125538ms step_avg:154.03ms
step:826/1480 train_time:125705ms step_avg:154.05ms
step:827/1480 train_time:125869ms step_avg:154.06ms
step:828/1480 train_time:126033ms step_avg:154.07ms
step:829/1480 train_time:126196ms step_avg:154.09ms
step:830/1480 train_time:126361ms step_avg:154.10ms
step:831/1480 train_time:126526ms step_avg:154.11ms
step:832/1480 train_time:126689ms step_avg:154.12ms
step:833/1480 train_time:126853ms step_avg:154.14ms
step:834/1480 train_time:127018ms step_avg:154.15ms
step:835/1480 train_time:127181ms step_avg:154.16ms
step:836/1480 train_time:127348ms step_avg:154.17ms
step:837/1480 train_time:127510ms step_avg:154.18ms
step:838/1480 train_time:127673ms step_avg:154.19ms
step:839/1480 train_time:127834ms step_avg:154.20ms
step:840/1480 train_time:127994ms step_avg:154.21ms
step:841/1480 train_time:128153ms step_avg:154.22ms
step:842/1480 train_time:128316ms step_avg:154.23ms
step:843/1480 train_time:128479ms step_avg:154.24ms
step:844/1480 train_time:128643ms step_avg:154.25ms
step:845/1480 train_time:128807ms step_avg:154.26ms
step:846/1480 train_time:128971ms step_avg:154.27ms
step:847/1480 train_time:129134ms step_avg:154.28ms
step:848/1480 train_time:129295ms step_avg:154.29ms
step:849/1480 train_time:129457ms step_avg:154.30ms
step:850/1480 train_time:129620ms step_avg:154.31ms
step:851/1480 train_time:129785ms step_avg:154.32ms
step:852/1480 train_time:129950ms step_avg:154.33ms
step:853/1480 train_time:130111ms step_avg:154.34ms
step:854/1480 train_time:130274ms step_avg:154.35ms
step:855/1480 train_time:130437ms step_avg:154.36ms
step:856/1480 train_time:130599ms step_avg:154.37ms
step:857/1480 train_time:130763ms step_avg:154.38ms
step:858/1480 train_time:130930ms step_avg:154.40ms
step:859/1480 train_time:131093ms step_avg:154.41ms
step:860/1480 train_time:131254ms step_avg:154.42ms
step:861/1480 train_time:131421ms step_avg:154.43ms
step:862/1480 train_time:131590ms step_avg:154.45ms
step:863/1480 train_time:131756ms step_avg:154.46ms
step:864/1480 train_time:131920ms step_avg:154.47ms
step:865/1480 train_time:132081ms step_avg:154.48ms
step:866/1480 train_time:132249ms step_avg:154.50ms
step:867/1480 train_time:132413ms step_avg:154.51ms
step:868/1480 train_time:132574ms step_avg:154.51ms
step:869/1480 train_time:132736ms step_avg:154.52ms
step:870/1480 train_time:132900ms step_avg:154.54ms
step:871/1480 train_time:133063ms step_avg:154.54ms
step:872/1480 train_time:133227ms step_avg:154.56ms
step:873/1480 train_time:133389ms step_avg:154.56ms
step:874/1480 train_time:133555ms step_avg:154.58ms
step:875/1480 train_time:133721ms step_avg:154.59ms
step:875/1480 val_loss:3.5031 train_time:133786ms step_avg:154.67ms
step:876/1480 train_time:133884ms step_avg:154.60ms
step:877/1480 train_time:134049ms step_avg:154.61ms
step:878/1480 train_time:134212ms step_avg:154.62ms
step:879/1480 train_time:134377ms step_avg:154.63ms
step:880/1480 train_time:134540ms step_avg:154.64ms
step:881/1480 train_time:134703ms step_avg:154.65ms
step:882/1480 train_time:134867ms step_avg:154.66ms
step:883/1480 train_time:135033ms step_avg:154.68ms
step:884/1480 train_time:135200ms step_avg:154.69ms
step:885/1480 train_time:135365ms step_avg:154.70ms
step:886/1480 train_time:135530ms step_avg:154.71ms
step:887/1480 train_time:135699ms step_avg:154.73ms
step:888/1480 train_time:135872ms step_avg:154.75ms
step:889/1480 train_time:136040ms step_avg:154.77ms
step:890/1480 train_time:136202ms step_avg:154.78ms
step:891/1480 train_time:136368ms step_avg:154.79ms
step:892/1480 train_time:136533ms step_avg:154.80ms
step:893/1480 train_time:136697ms step_avg:154.81ms
step:894/1480 train_time:136864ms step_avg:154.82ms
step:895/1480 train_time:137028ms step_avg:154.83ms
step:896/1480 train_time:137194ms step_avg:154.85ms
step:897/1480 train_time:137360ms step_avg:154.86ms
step:898/1480 train_time:137526ms step_avg:154.87ms
step:899/1480 train_time:137688ms step_avg:154.88ms
step:900/1480 train_time:137852ms step_avg:154.89ms
step:901/1480 train_time:138017ms step_avg:154.90ms
step:902/1480 train_time:138182ms step_avg:154.91ms
step:903/1480 train_time:138352ms step_avg:154.93ms
step:904/1480 train_time:138518ms step_avg:154.94ms
step:905/1480 train_time:138681ms step_avg:154.95ms
step:906/1480 train_time:138847ms step_avg:154.96ms
step:907/1480 train_time:139016ms step_avg:154.98ms
step:908/1480 train_time:139179ms step_avg:154.99ms
step:909/1480 train_time:139344ms step_avg:155.00ms
step:910/1480 train_time:139515ms step_avg:155.02ms
step:911/1480 train_time:139681ms step_avg:155.03ms
step:912/1480 train_time:139846ms step_avg:155.04ms
step:913/1480 train_time:140013ms step_avg:155.05ms
step:914/1480 train_time:140181ms step_avg:155.07ms
step:915/1480 train_time:140350ms step_avg:155.08ms
step:916/1480 train_time:140516ms step_avg:155.09ms
step:917/1480 train_time:140680ms step_avg:155.11ms
step:918/1480 train_time:140848ms step_avg:155.12ms
step:919/1480 train_time:141017ms step_avg:155.13ms
step:920/1480 train_time:141183ms step_avg:155.15ms
step:921/1480 train_time:141349ms step_avg:155.16ms
step:922/1480 train_time:141516ms step_avg:155.17ms
step:923/1480 train_time:141679ms step_avg:155.18ms
step:924/1480 train_time:141844ms step_avg:155.19ms
step:925/1480 train_time:142009ms step_avg:155.20ms
step:926/1480 train_time:142172ms step_avg:155.21ms
step:927/1480 train_time:142337ms step_avg:155.22ms
step:928/1480 train_time:142502ms step_avg:155.23ms
step:929/1480 train_time:142666ms step_avg:155.24ms
step:930/1480 train_time:142830ms step_avg:155.25ms
step:931/1480 train_time:142994ms step_avg:155.26ms
step:932/1480 train_time:143161ms step_avg:155.27ms
step:933/1480 train_time:143330ms step_avg:155.29ms
step:934/1480 train_time:143498ms step_avg:155.30ms
step:935/1480 train_time:143668ms step_avg:155.32ms
step:936/1480 train_time:143835ms step_avg:155.33ms
step:937/1480 train_time:144004ms step_avg:155.34ms
step:938/1480 train_time:144166ms step_avg:155.35ms
step:939/1480 train_time:144335ms step_avg:155.37ms
step:940/1480 train_time:144502ms step_avg:155.38ms
step:941/1480 train_time:144666ms step_avg:155.39ms
step:942/1480 train_time:144830ms step_avg:155.40ms
step:943/1480 train_time:145002ms step_avg:155.41ms
step:944/1480 train_time:145176ms step_avg:155.43ms
step:945/1480 train_time:145339ms step_avg:155.44ms
step:946/1480 train_time:145507ms step_avg:155.46ms
step:947/1480 train_time:145675ms step_avg:155.47ms
step:948/1480 train_time:145841ms step_avg:155.48ms
step:949/1480 train_time:146005ms step_avg:155.49ms
step:950/1480 train_time:146170ms step_avg:155.50ms
step:951/1480 train_time:146339ms step_avg:155.51ms
step:952/1480 train_time:146504ms step_avg:155.52ms
step:953/1480 train_time:146673ms step_avg:155.54ms
step:954/1480 train_time:146843ms step_avg:155.55ms
step:955/1480 train_time:147006ms step_avg:155.56ms
step:956/1480 train_time:147173ms step_avg:155.57ms
step:957/1480 train_time:147340ms step_avg:155.59ms
step:958/1480 train_time:147509ms step_avg:155.60ms
step:959/1480 train_time:147674ms step_avg:155.61ms
step:960/1480 train_time:147841ms step_avg:155.62ms
step:961/1480 train_time:148005ms step_avg:155.63ms
step:962/1480 train_time:148168ms step_avg:155.64ms
step:963/1480 train_time:148334ms step_avg:155.65ms
step:964/1480 train_time:148503ms step_avg:155.66ms
step:965/1480 train_time:148665ms step_avg:155.67ms
step:966/1480 train_time:148830ms step_avg:155.68ms
step:967/1480 train_time:148994ms step_avg:155.69ms
step:968/1480 train_time:149160ms step_avg:155.70ms
step:969/1480 train_time:149326ms step_avg:155.71ms
step:970/1480 train_time:149489ms step_avg:155.72ms
step:971/1480 train_time:149654ms step_avg:155.73ms
step:972/1480 train_time:149819ms step_avg:155.74ms
step:973/1480 train_time:149983ms step_avg:155.75ms
step:974/1480 train_time:150151ms step_avg:155.76ms
step:975/1480 train_time:150317ms step_avg:155.77ms
step:976/1480 train_time:150482ms step_avg:155.78ms
step:977/1480 train_time:150645ms step_avg:155.79ms
step:978/1480 train_time:150810ms step_avg:155.80ms
step:979/1480 train_time:150978ms step_avg:155.81ms
step:980/1480 train_time:151143ms step_avg:155.82ms
step:981/1480 train_time:151311ms step_avg:155.83ms
step:982/1480 train_time:151476ms step_avg:155.84ms
step:983/1480 train_time:151640ms step_avg:155.85ms
step:984/1480 train_time:151804ms step_avg:155.86ms
step:985/1480 train_time:151971ms step_avg:155.87ms
step:986/1480 train_time:152138ms step_avg:155.88ms
step:987/1480 train_time:152302ms step_avg:155.89ms
step:988/1480 train_time:152469ms step_avg:155.90ms
step:989/1480 train_time:152635ms step_avg:155.91ms
step:990/1480 train_time:152804ms step_avg:155.92ms
step:991/1480 train_time:152972ms step_avg:155.93ms
step:992/1480 train_time:153146ms step_avg:155.95ms
step:993/1480 train_time:153320ms step_avg:155.97ms
step:994/1480 train_time:153484ms step_avg:155.98ms
step:995/1480 train_time:153647ms step_avg:155.99ms
step:996/1480 train_time:153810ms step_avg:155.99ms
step:997/1480 train_time:153976ms step_avg:156.00ms
step:998/1480 train_time:154139ms step_avg:156.01ms
step:999/1480 train_time:154304ms step_avg:156.02ms
step:1000/1480 train_time:154474ms step_avg:156.03ms
step:1000/1480 val_loss:3.4400 train_time:154541ms step_avg:156.10ms
step:1001/1480 train_time:154640ms step_avg:156.04ms
step:1002/1480 train_time:154807ms step_avg:156.06ms
step:1003/1480 train_time:154977ms step_avg:156.07ms
step:1004/1480 train_time:155145ms step_avg:156.08ms
step:1005/1480 train_time:155313ms step_avg:156.09ms
step:1006/1480 train_time:155480ms step_avg:156.10ms
step:1007/1480 train_time:155645ms step_avg:156.11ms
step:1008/1480 train_time:155814ms step_avg:156.13ms
step:1009/1480 train_time:155986ms step_avg:156.14ms
step:1010/1480 train_time:156153ms step_avg:156.15ms
step:1011/1480 train_time:156318ms step_avg:156.16ms
step:1012/1480 train_time:156482ms step_avg:156.17ms
step:1013/1480 train_time:156652ms step_avg:156.18ms
step:1014/1480 train_time:156819ms step_avg:156.19ms
step:1015/1480 train_time:156990ms step_avg:156.21ms
step:1016/1480 train_time:157160ms step_avg:156.22ms
step:1017/1480 train_time:157332ms step_avg:156.24ms
step:1018/1480 train_time:157500ms step_avg:156.25ms
step:1019/1480 train_time:157670ms step_avg:156.26ms
step:1020/1480 train_time:157839ms step_avg:156.28ms
step:1021/1480 train_time:158004ms step_avg:156.28ms
step:1022/1480 train_time:158170ms step_avg:156.29ms
step:1023/1480 train_time:158336ms step_avg:156.30ms
step:1024/1480 train_time:158502ms step_avg:156.31ms
step:1025/1480 train_time:158673ms step_avg:156.33ms
step:1026/1480 train_time:158839ms step_avg:156.34ms
step:1027/1480 train_time:159005ms step_avg:156.35ms
step:1028/1480 train_time:159178ms step_avg:156.36ms
step:1029/1480 train_time:159353ms step_avg:156.38ms
step:1030/1480 train_time:159521ms step_avg:156.39ms
step:1031/1480 train_time:159684ms step_avg:156.40ms
step:1032/1480 train_time:159858ms step_avg:156.42ms
step:1033/1480 train_time:160023ms step_avg:156.43ms
step:1034/1480 train_time:160192ms step_avg:156.44ms
step:1035/1480 train_time:160359ms step_avg:156.45ms
step:1036/1480 train_time:160524ms step_avg:156.46ms
step:1037/1480 train_time:160691ms step_avg:156.47ms
step:1038/1480 train_time:160858ms step_avg:156.48ms
step:1039/1480 train_time:161028ms step_avg:156.49ms
step:1040/1480 train_time:161196ms step_avg:156.50ms
step:1041/1480 train_time:161363ms step_avg:156.51ms
step:1042/1480 train_time:161528ms step_avg:156.52ms
step:1043/1480 train_time:161694ms step_avg:156.53ms
step:1044/1480 train_time:161859ms step_avg:156.54ms
step:1045/1480 train_time:162031ms step_avg:156.55ms
step:1046/1480 train_time:162199ms step_avg:156.56ms
step:1047/1480 train_time:162365ms step_avg:156.57ms
step:1048/1480 train_time:162531ms step_avg:156.58ms
step:1049/1480 train_time:162697ms step_avg:156.59ms
step:1050/1480 train_time:162866ms step_avg:156.60ms
step:1051/1480 train_time:163036ms step_avg:156.61ms
step:1052/1480 train_time:163202ms step_avg:156.62ms
step:1053/1480 train_time:163369ms step_avg:156.63ms
step:1054/1480 train_time:163538ms step_avg:156.65ms
step:1055/1480 train_time:163703ms step_avg:156.65ms
step:1056/1480 train_time:163868ms step_avg:156.66ms
step:1057/1480 train_time:164034ms step_avg:156.67ms
step:1058/1480 train_time:164202ms step_avg:156.68ms
step:1059/1480 train_time:164375ms step_avg:156.70ms
step:1060/1480 train_time:164543ms step_avg:156.71ms
step:1061/1480 train_time:164707ms step_avg:156.71ms
step:1062/1480 train_time:164873ms step_avg:156.72ms
step:1063/1480 train_time:165039ms step_avg:156.73ms
step:1064/1480 train_time:165203ms step_avg:156.74ms
step:1065/1480 train_time:165370ms step_avg:156.75ms
step:1066/1480 train_time:165538ms step_avg:156.76ms
step:1067/1480 train_time:165707ms step_avg:156.77ms
step:1068/1480 train_time:165874ms step_avg:156.78ms
step:1069/1480 train_time:166044ms step_avg:156.79ms
step:1070/1480 train_time:166209ms step_avg:156.80ms
step:1071/1480 train_time:166381ms step_avg:156.82ms
step:1072/1480 train_time:166549ms step_avg:156.83ms
step:1073/1480 train_time:166712ms step_avg:156.83ms
step:1074/1480 train_time:166878ms step_avg:156.84ms
step:1075/1480 train_time:167048ms step_avg:156.85ms
step:1076/1480 train_time:167218ms step_avg:156.86ms
step:1077/1480 train_time:167382ms step_avg:156.87ms
step:1078/1480 train_time:167556ms step_avg:156.89ms
step:1079/1480 train_time:167729ms step_avg:156.90ms
step:1080/1480 train_time:167899ms step_avg:156.92ms
step:1081/1480 train_time:168066ms step_avg:156.92ms
step:1082/1480 train_time:168233ms step_avg:156.93ms
step:1083/1480 train_time:168400ms step_avg:156.94ms
step:1084/1480 train_time:168565ms step_avg:156.95ms
step:1085/1480 train_time:168734ms step_avg:156.96ms
step:1086/1480 train_time:168901ms step_avg:156.97ms
step:1087/1480 train_time:169066ms step_avg:156.98ms
step:1088/1480 train_time:169238ms step_avg:156.99ms
step:1089/1480 train_time:169410ms step_avg:157.01ms
step:1090/1480 train_time:169582ms step_avg:157.02ms
step:1091/1480 train_time:169752ms step_avg:157.03ms
step:1092/1480 train_time:169919ms step_avg:157.04ms
step:1093/1480 train_time:170087ms step_avg:157.05ms
step:1094/1480 train_time:170253ms step_avg:157.06ms
step:1095/1480 train_time:170417ms step_avg:157.07ms
step:1096/1480 train_time:170586ms step_avg:157.08ms
step:1097/1480 train_time:170755ms step_avg:157.09ms
step:1098/1480 train_time:170927ms step_avg:157.10ms
step:1099/1480 train_time:171099ms step_avg:157.12ms
step:1100/1480 train_time:171271ms step_avg:157.13ms
step:1101/1480 train_time:171440ms step_avg:157.14ms
step:1102/1480 train_time:171612ms step_avg:157.15ms
step:1103/1480 train_time:171790ms step_avg:157.17ms
step:1104/1480 train_time:171959ms step_avg:157.18ms
step:1105/1480 train_time:172129ms step_avg:157.20ms
step:1106/1480 train_time:172299ms step_avg:157.21ms
step:1107/1480 train_time:172469ms step_avg:157.22ms
step:1108/1480 train_time:172635ms step_avg:157.23ms
step:1109/1480 train_time:172800ms step_avg:157.23ms
step:1110/1480 train_time:172964ms step_avg:157.24ms
step:1111/1480 train_time:173133ms step_avg:157.25ms
step:1112/1480 train_time:173303ms step_avg:157.26ms
step:1113/1480 train_time:173483ms step_avg:157.28ms
step:1114/1480 train_time:173656ms step_avg:157.30ms
step:1115/1480 train_time:173829ms step_avg:157.31ms
step:1116/1480 train_time:173996ms step_avg:157.32ms
step:1117/1480 train_time:174169ms step_avg:157.33ms
step:1118/1480 train_time:174345ms step_avg:157.35ms
step:1119/1480 train_time:174513ms step_avg:157.36ms
step:1120/1480 train_time:174680ms step_avg:157.37ms
step:1121/1480 train_time:174849ms step_avg:157.38ms
step:1122/1480 train_time:175018ms step_avg:157.39ms
step:1123/1480 train_time:175184ms step_avg:157.40ms
step:1124/1480 train_time:175352ms step_avg:157.41ms
step:1125/1480 train_time:175520ms step_avg:157.42ms
step:1125/1480 val_loss:3.3838 train_time:175587ms step_avg:157.48ms
step:1126/1480 train_time:175688ms step_avg:157.43ms
step:1127/1480 train_time:175860ms step_avg:157.44ms
step:1128/1480 train_time:176031ms step_avg:157.45ms
step:1129/1480 train_time:176203ms step_avg:157.46ms
step:1130/1480 train_time:176373ms step_avg:157.48ms
step:1131/1480 train_time:176552ms step_avg:157.50ms
step:1132/1480 train_time:176718ms step_avg:157.50ms
step:1133/1480 train_time:176890ms step_avg:157.52ms
step:1134/1480 train_time:177061ms step_avg:157.53ms
step:1135/1480 train_time:177229ms step_avg:157.54ms
step:1136/1480 train_time:177400ms step_avg:157.55ms
step:1137/1480 train_time:177570ms step_avg:157.56ms
step:1138/1480 train_time:177742ms step_avg:157.57ms
step:1139/1480 train_time:177909ms step_avg:157.58ms
step:1140/1480 train_time:178077ms step_avg:157.59ms
step:1141/1480 train_time:178248ms step_avg:157.60ms
step:1142/1480 train_time:178416ms step_avg:157.61ms
step:1143/1480 train_time:178587ms step_avg:157.62ms
step:1144/1480 train_time:178755ms step_avg:157.63ms
step:1145/1480 train_time:178921ms step_avg:157.64ms
step:1146/1480 train_time:179092ms step_avg:157.65ms
step:1147/1480 train_time:179261ms step_avg:157.66ms
step:1148/1480 train_time:179430ms step_avg:157.67ms
step:1149/1480 train_time:179601ms step_avg:157.68ms
step:1150/1480 train_time:179770ms step_avg:157.69ms
step:1151/1480 train_time:179943ms step_avg:157.71ms
step:1152/1480 train_time:180115ms step_avg:157.72ms
step:1153/1480 train_time:180287ms step_avg:157.73ms
step:1154/1480 train_time:180454ms step_avg:157.74ms
step:1155/1480 train_time:180626ms step_avg:157.75ms
step:1156/1480 train_time:180803ms step_avg:157.77ms
step:1157/1480 train_time:180973ms step_avg:157.78ms
step:1158/1480 train_time:181141ms step_avg:157.79ms
step:1159/1480 train_time:181307ms step_avg:157.80ms
step:1160/1480 train_time:181473ms step_avg:157.80ms
step:1161/1480 train_time:181647ms step_avg:157.82ms
step:1162/1480 train_time:181818ms step_avg:157.83ms
step:1163/1480 train_time:181987ms step_avg:157.84ms
step:1164/1480 train_time:182157ms step_avg:157.85ms
step:1165/1480 train_time:182323ms step_avg:157.86ms
step:1166/1480 train_time:182491ms step_avg:157.86ms
step:1167/1480 train_time:182660ms step_avg:157.87ms
step:1168/1480 train_time:182828ms step_avg:157.88ms
step:1169/1480 train_time:182997ms step_avg:157.89ms
step:1170/1480 train_time:183165ms step_avg:157.90ms
step:1171/1480 train_time:183334ms step_avg:157.91ms
step:1172/1480 train_time:183499ms step_avg:157.92ms
step:1173/1480 train_time:183670ms step_avg:157.93ms
step:1174/1480 train_time:183854ms step_avg:157.95ms
step:1175/1480 train_time:184024ms step_avg:157.96ms
step:1176/1480 train_time:184197ms step_avg:157.97ms
step:1177/1480 train_time:184374ms step_avg:157.99ms
step:1178/1480 train_time:184542ms step_avg:158.00ms
step:1179/1480 train_time:184707ms step_avg:158.00ms
step:1180/1480 train_time:184888ms step_avg:158.02ms
step:1181/1480 train_time:185059ms step_avg:158.03ms
step:1182/1480 train_time:185225ms step_avg:158.04ms
step:1183/1480 train_time:185395ms step_avg:158.05ms
step:1184/1480 train_time:185563ms step_avg:158.06ms
step:1185/1480 train_time:185736ms step_avg:158.07ms
step:1186/1480 train_time:185907ms step_avg:158.08ms
step:1187/1480 train_time:186091ms step_avg:158.11ms
step:1188/1480 train_time:186258ms step_avg:158.11ms
step:1189/1480 train_time:186427ms step_avg:158.12ms
step:1190/1480 train_time:186595ms step_avg:158.13ms
step:1191/1480 train_time:186766ms step_avg:158.14ms
step:1192/1480 train_time:186933ms step_avg:158.15ms
step:1193/1480 train_time:187099ms step_avg:158.16ms
step:1194/1480 train_time:187267ms step_avg:158.16ms
step:1195/1480 train_time:187441ms step_avg:158.18ms
step:1196/1480 train_time:187623ms step_avg:158.20ms
step:1197/1480 train_time:187795ms step_avg:158.21ms
step:1198/1480 train_time:187977ms step_avg:158.23ms
step:1199/1480 train_time:188147ms step_avg:158.24ms
step:1200/1480 train_time:188317ms step_avg:158.25ms
step:1201/1480 train_time:188483ms step_avg:158.26ms
step:1202/1480 train_time:188666ms step_avg:158.28ms
step:1203/1480 train_time:188842ms step_avg:158.29ms
step:1204/1480 train_time:189016ms step_avg:158.30ms
step:1205/1480 train_time:189184ms step_avg:158.31ms
step:1206/1480 train_time:189353ms step_avg:158.32ms
step:1207/1480 train_time:189523ms step_avg:158.33ms
step:1208/1480 train_time:189689ms step_avg:158.34ms
step:1209/1480 train_time:189863ms step_avg:158.35ms
step:1210/1480 train_time:190039ms step_avg:158.37ms
step:1211/1480 train_time:190211ms step_avg:158.38ms
step:1212/1480 train_time:190382ms step_avg:158.39ms
step:1213/1480 train_time:190554ms step_avg:158.40ms
step:1214/1480 train_time:190730ms step_avg:158.41ms
step:1215/1480 train_time:190902ms step_avg:158.43ms
step:1216/1480 train_time:191070ms step_avg:158.43ms
step:1217/1480 train_time:191244ms step_avg:158.45ms
step:1218/1480 train_time:191413ms step_avg:158.45ms
step:1219/1480 train_time:191593ms step_avg:158.47ms
step:1220/1480 train_time:191762ms step_avg:158.48ms
step:1221/1480 train_time:191931ms step_avg:158.49ms
step:1222/1480 train_time:192098ms step_avg:158.50ms
step:1223/1480 train_time:192268ms step_avg:158.51ms
step:1224/1480 train_time:192446ms step_avg:158.52ms
step:1225/1480 train_time:192618ms step_avg:158.53ms
step:1226/1480 train_time:192791ms step_avg:158.55ms
step:1227/1480 train_time:192964ms step_avg:158.56ms
step:1228/1480 train_time:193133ms step_avg:158.57ms
step:1229/1480 train_time:193305ms step_avg:158.58ms
step:1230/1480 train_time:193485ms step_avg:158.59ms
step:1231/1480 train_time:193661ms step_avg:158.61ms
step:1232/1480 train_time:193836ms step_avg:158.62ms
step:1233/1480 train_time:194006ms step_avg:158.63ms
step:1234/1480 train_time:194176ms step_avg:158.64ms
step:1235/1480 train_time:194350ms step_avg:158.65ms
step:1236/1480 train_time:194519ms step_avg:158.66ms
step:1237/1480 train_time:194691ms step_avg:158.67ms
step:1238/1480 train_time:194876ms step_avg:158.69ms
step:1239/1480 train_time:195047ms step_avg:158.70ms
step:1240/1480 train_time:195218ms step_avg:158.71ms
step:1241/1480 train_time:195392ms step_avg:158.73ms
step:1242/1480 train_time:195562ms step_avg:158.73ms
step:1243/1480 train_time:195735ms step_avg:158.75ms
step:1244/1480 train_time:195902ms step_avg:158.75ms
step:1245/1480 train_time:196070ms step_avg:158.76ms
step:1246/1480 train_time:196241ms step_avg:158.77ms
step:1247/1480 train_time:196410ms step_avg:158.78ms
step:1248/1480 train_time:196580ms step_avg:158.79ms
step:1249/1480 train_time:196748ms step_avg:158.80ms
step:1250/1480 train_time:196919ms step_avg:158.81ms
step:1250/1480 val_loss:3.3340 train_time:196990ms step_avg:158.86ms
step:1251/1480 train_time:197099ms step_avg:158.82ms
step:1252/1480 train_time:197267ms step_avg:158.83ms
step:1253/1480 train_time:197437ms step_avg:158.84ms
step:1254/1480 train_time:197609ms step_avg:158.85ms
step:1255/1480 train_time:197795ms step_avg:158.87ms
step:1256/1480 train_time:197968ms step_avg:158.88ms
step:1257/1480 train_time:198138ms step_avg:158.89ms
step:1258/1480 train_time:198312ms step_avg:158.90ms
step:1259/1480 train_time:198483ms step_avg:158.91ms
step:1260/1480 train_time:198650ms step_avg:158.92ms
step:1261/1480 train_time:198821ms step_avg:158.93ms
step:1262/1480 train_time:198998ms step_avg:158.94ms
step:1263/1480 train_time:199172ms step_avg:158.96ms
step:1264/1480 train_time:199339ms step_avg:158.96ms
step:1265/1480 train_time:199506ms step_avg:158.97ms
step:1266/1480 train_time:199679ms step_avg:158.98ms
step:1267/1480 train_time:199849ms step_avg:158.99ms
step:1268/1480 train_time:200021ms step_avg:159.00ms
step:1269/1480 train_time:200196ms step_avg:159.01ms
step:1270/1480 train_time:200365ms step_avg:159.02ms
step:1271/1480 train_time:200537ms step_avg:159.03ms
step:1272/1480 train_time:200702ms step_avg:159.04ms
step:1273/1480 train_time:200873ms step_avg:159.04ms
step:1274/1480 train_time:201045ms step_avg:159.05ms
step:1275/1480 train_time:201212ms step_avg:159.06ms
step:1276/1480 train_time:201379ms step_avg:159.07ms
step:1277/1480 train_time:201551ms step_avg:159.08ms
step:1278/1480 train_time:201718ms step_avg:159.08ms
step:1279/1480 train_time:201891ms step_avg:159.09ms
step:1280/1480 train_time:202071ms step_avg:159.11ms
step:1281/1480 train_time:202239ms step_avg:159.12ms
step:1282/1480 train_time:202406ms step_avg:159.12ms
step:1283/1480 train_time:202576ms step_avg:159.13ms
step:1284/1480 train_time:202746ms step_avg:159.14ms
step:1285/1480 train_time:202916ms step_avg:159.15ms
step:1286/1480 train_time:203087ms step_avg:159.16ms
step:1287/1480 train_time:203258ms step_avg:159.17ms
step:1288/1480 train_time:203429ms step_avg:159.18ms
step:1289/1480 train_time:203614ms step_avg:159.20ms
step:1290/1480 train_time:203794ms step_avg:159.21ms
step:1291/1480 train_time:203969ms step_avg:159.23ms
step:1292/1480 train_time:204144ms step_avg:159.24ms
step:1293/1480 train_time:204318ms step_avg:159.25ms
step:1294/1480 train_time:204489ms step_avg:159.26ms
step:1295/1480 train_time:204660ms step_avg:159.27ms
step:1296/1480 train_time:204833ms step_avg:159.28ms
step:1297/1480 train_time:205004ms step_avg:159.29ms
step:1298/1480 train_time:205174ms step_avg:159.30ms
step:1299/1480 train_time:205345ms step_avg:159.31ms
step:1300/1480 train_time:205511ms step_avg:159.31ms
step:1301/1480 train_time:205682ms step_avg:159.32ms
step:1302/1480 train_time:205854ms step_avg:159.33ms
step:1303/1480 train_time:206029ms step_avg:159.34ms
step:1304/1480 train_time:206203ms step_avg:159.35ms
step:1305/1480 train_time:206371ms step_avg:159.36ms
step:1306/1480 train_time:206547ms step_avg:159.37ms
step:1307/1480 train_time:206714ms step_avg:159.38ms
step:1308/1480 train_time:206885ms step_avg:159.39ms
step:1309/1480 train_time:207059ms step_avg:159.40ms
step:1310/1480 train_time:207227ms step_avg:159.41ms
step:1311/1480 train_time:207395ms step_avg:159.41ms
step:1312/1480 train_time:207569ms step_avg:159.42ms
step:1313/1480 train_time:207738ms step_avg:159.43ms
step:1314/1480 train_time:207913ms step_avg:159.44ms
step:1315/1480 train_time:208084ms step_avg:159.45ms
step:1316/1480 train_time:208249ms step_avg:159.46ms
step:1317/1480 train_time:208422ms step_avg:159.47ms
step:1318/1480 train_time:208603ms step_avg:159.48ms
step:1319/1480 train_time:208780ms step_avg:159.50ms
step:1320/1480 train_time:208958ms step_avg:159.51ms
step:1321/1480 train_time:209130ms step_avg:159.52ms
step:1322/1480 train_time:209313ms step_avg:159.54ms
step:1323/1480 train_time:209487ms step_avg:159.55ms
step:1324/1480 train_time:209662ms step_avg:159.56ms
step:1325/1480 train_time:209844ms step_avg:159.58ms
step:1326/1480 train_time:210019ms step_avg:159.59ms
step:1327/1480 train_time:210188ms step_avg:159.60ms
step:1328/1480 train_time:210359ms step_avg:159.60ms
step:1329/1480 train_time:210555ms step_avg:159.63ms
step:1330/1480 train_time:210733ms step_avg:159.65ms
step:1331/1480 train_time:210903ms step_avg:159.65ms
step:1332/1480 train_time:211078ms step_avg:159.67ms
step:1333/1480 train_time:211252ms step_avg:159.68ms
step:1334/1480 train_time:211424ms step_avg:159.69ms
step:1335/1480 train_time:211592ms step_avg:159.69ms
step:1336/1480 train_time:211777ms step_avg:159.71ms
step:1337/1480 train_time:211951ms step_avg:159.72ms
step:1338/1480 train_time:212123ms step_avg:159.73ms
step:1339/1480 train_time:212296ms step_avg:159.74ms
step:1340/1480 train_time:212469ms step_avg:159.75ms
step:1341/1480 train_time:212637ms step_avg:159.76ms
step:1342/1480 train_time:212811ms step_avg:159.77ms
step:1343/1480 train_time:212982ms step_avg:159.78ms
step:1344/1480 train_time:213153ms step_avg:159.78ms
step:1345/1480 train_time:213332ms step_avg:159.80ms
step:1346/1480 train_time:213500ms step_avg:159.81ms
step:1347/1480 train_time:213670ms step_avg:159.81ms
step:1348/1480 train_time:213840ms step_avg:159.82ms
step:1349/1480 train_time:214009ms step_avg:159.83ms
step:1350/1480 train_time:214185ms step_avg:159.84ms
step:1351/1480 train_time:214356ms step_avg:159.85ms
step:1352/1480 train_time:214526ms step_avg:159.86ms
step:1353/1480 train_time:214703ms step_avg:159.87ms
step:1354/1480 train_time:214875ms step_avg:159.88ms
step:1355/1480 train_time:215043ms step_avg:159.88ms
step:1356/1480 train_time:215214ms step_avg:159.89ms
step:1357/1480 train_time:215387ms step_avg:159.90ms
step:1358/1480 train_time:215559ms step_avg:159.91ms
step:1359/1480 train_time:215731ms step_avg:159.92ms
step:1360/1480 train_time:215905ms step_avg:159.93ms
step:1361/1480 train_time:216083ms step_avg:159.94ms
step:1362/1480 train_time:216259ms step_avg:159.96ms
step:1363/1480 train_time:216441ms step_avg:159.97ms
step:1364/1480 train_time:216609ms step_avg:159.98ms
step:1365/1480 train_time:216777ms step_avg:159.98ms
step:1366/1480 train_time:216948ms step_avg:159.99ms
step:1367/1480 train_time:217118ms step_avg:160.00ms
step:1368/1480 train_time:217291ms step_avg:160.01ms
step:1369/1480 train_time:217474ms step_avg:160.02ms
step:1370/1480 train_time:217650ms step_avg:160.04ms
step:1371/1480 train_time:217822ms step_avg:160.05ms
step:1372/1480 train_time:218000ms step_avg:160.06ms
step:1373/1480 train_time:218168ms step_avg:160.06ms
step:1374/1480 train_time:218345ms step_avg:160.08ms
step:1375/1480 train_time:218516ms step_avg:160.08ms
step:1375/1480 val_loss:3.2953 train_time:218583ms step_avg:160.13ms
step:1376/1480 train_time:218687ms step_avg:160.09ms
step:1377/1480 train_time:218861ms step_avg:160.10ms
step:1378/1480 train_time:219028ms step_avg:160.11ms
step:1379/1480 train_time:219204ms step_avg:160.12ms
step:1380/1480 train_time:219378ms step_avg:160.13ms
step:1381/1480 train_time:219559ms step_avg:160.15ms
step:1382/1480 train_time:219731ms step_avg:160.15ms
step:1383/1480 train_time:219903ms step_avg:160.16ms
step:1384/1480 train_time:220080ms step_avg:160.17ms
step:1385/1480 train_time:220246ms step_avg:160.18ms
step:1386/1480 train_time:220416ms step_avg:160.19ms
step:1387/1480 train_time:220587ms step_avg:160.19ms
step:1388/1480 train_time:220756ms step_avg:160.20ms
step:1389/1480 train_time:220929ms step_avg:160.21ms
step:1390/1480 train_time:221098ms step_avg:160.22ms
step:1391/1480 train_time:221267ms step_avg:160.22ms
step:1392/1480 train_time:221440ms step_avg:160.23ms
step:1393/1480 train_time:221611ms step_avg:160.24ms
step:1394/1480 train_time:221782ms step_avg:160.25ms
step:1395/1480 train_time:221951ms step_avg:160.25ms
step:1396/1480 train_time:222120ms step_avg:160.26ms
step:1397/1480 train_time:222287ms step_avg:160.26ms
step:1398/1480 train_time:222453ms step_avg:160.27ms
step:1399/1480 train_time:222622ms step_avg:160.28ms
step:1400/1480 train_time:222800ms step_avg:160.29ms
step:1401/1480 train_time:222967ms step_avg:160.29ms
step:1402/1480 train_time:223139ms step_avg:160.30ms
step:1403/1480 train_time:223315ms step_avg:160.31ms
step:1404/1480 train_time:223486ms step_avg:160.32ms
step:1405/1480 train_time:223661ms step_avg:160.33ms
step:1406/1480 train_time:223836ms step_avg:160.34ms
step:1407/1480 train_time:224005ms step_avg:160.35ms
step:1408/1480 train_time:224173ms step_avg:160.35ms
step:1409/1480 train_time:224359ms step_avg:160.37ms
step:1410/1480 train_time:224528ms step_avg:160.38ms
step:1411/1480 train_time:224698ms step_avg:160.38ms
step:1412/1480 train_time:224867ms step_avg:160.39ms
step:1413/1480 train_time:225038ms step_avg:160.40ms
step:1414/1480 train_time:225209ms step_avg:160.40ms
step:1415/1480 train_time:225385ms step_avg:160.42ms
step:1416/1480 train_time:225570ms step_avg:160.43ms
step:1417/1480 train_time:225744ms step_avg:160.44ms
step:1418/1480 train_time:225916ms step_avg:160.45ms
step:1419/1480 train_time:226089ms step_avg:160.46ms
step:1420/1480 train_time:226263ms step_avg:160.47ms
step:1421/1480 train_time:226436ms step_avg:160.48ms
step:1422/1480 train_time:226607ms step_avg:160.49ms
step:1423/1480 train_time:226777ms step_avg:160.49ms
step:1424/1480 train_time:226956ms step_avg:160.51ms
step:1425/1480 train_time:227136ms step_avg:160.52ms
step:1426/1480 train_time:227306ms step_avg:160.53ms
step:1427/1480 train_time:227482ms step_avg:160.54ms
step:1428/1480 train_time:227651ms step_avg:160.54ms
step:1429/1480 train_time:227818ms step_avg:160.55ms
step:1430/1480 train_time:227993ms step_avg:160.56ms
step:1431/1480 train_time:228167ms step_avg:160.57ms
step:1432/1480 train_time:228344ms step_avg:160.58ms
step:1433/1480 train_time:228523ms step_avg:160.59ms
step:1434/1480 train_time:228704ms step_avg:160.61ms
step:1435/1480 train_time:228880ms step_avg:160.62ms
step:1436/1480 train_time:229055ms step_avg:160.63ms
step:1437/1480 train_time:229225ms step_avg:160.63ms
step:1438/1480 train_time:229393ms step_avg:160.64ms
step:1439/1480 train_time:229567ms step_avg:160.65ms
step:1440/1480 train_time:229739ms step_avg:160.66ms
step:1441/1480 train_time:229909ms step_avg:160.66ms
step:1442/1480 train_time:230087ms step_avg:160.68ms
step:1443/1480 train_time:230276ms step_avg:160.70ms
step:1444/1480 train_time:230448ms step_avg:160.70ms
step:1445/1480 train_time:230621ms step_avg:160.71ms
step:1446/1480 train_time:230798ms step_avg:160.72ms
step:1447/1480 train_time:230975ms step_avg:160.73ms
step:1448/1480 train_time:231145ms step_avg:160.74ms
step:1449/1480 train_time:231319ms step_avg:160.75ms
step:1450/1480 train_time:231491ms step_avg:160.76ms
step:1451/1480 train_time:231663ms step_avg:160.77ms
step:1452/1480 train_time:231836ms step_avg:160.77ms
step:1453/1480 train_time:232004ms step_avg:160.78ms
step:1454/1480 train_time:232175ms step_avg:160.79ms
step:1455/1480 train_time:232352ms step_avg:160.80ms
step:1456/1480 train_time:232525ms step_avg:160.81ms
step:1457/1480 train_time:232696ms step_avg:160.81ms
step:1458/1480 train_time:232866ms step_avg:160.82ms
step:1459/1480 train_time:233042ms step_avg:160.83ms
step:1460/1480 train_time:233213ms step_avg:160.84ms
step:1461/1480 train_time:233388ms step_avg:160.85ms
step:1462/1480 train_time:233559ms step_avg:160.85ms
step:1463/1480 train_time:233737ms step_avg:160.86ms
step:1464/1480 train_time:233910ms step_avg:160.87ms
step:1465/1480 train_time:234084ms step_avg:160.88ms
step:1466/1480 train_time:234254ms step_avg:160.89ms
step:1467/1480 train_time:234429ms step_avg:160.90ms
step:1468/1480 train_time:234600ms step_avg:160.91ms
step:1469/1480 train_time:234773ms step_avg:160.91ms
step:1470/1480 train_time:234954ms step_avg:160.93ms
step:1471/1480 train_time:235143ms step_avg:160.95ms
step:1472/1480 train_time:235323ms step_avg:160.96ms
step:1473/1480 train_time:235494ms step_avg:160.97ms
step:1474/1480 train_time:235670ms step_avg:160.98ms
step:1475/1480 train_time:235849ms step_avg:160.99ms
step:1476/1480 train_time:236021ms step_avg:161.00ms
step:1477/1480 train_time:236205ms step_avg:161.01ms
step:1478/1480 train_time:236385ms step_avg:161.03ms
step:1479/1480 train_time:236561ms step_avg:161.04ms
step:1480/1480 train_time:236734ms step_avg:161.04ms
step:1480/1480 val_loss:3.2764 train_time:236804ms step_avg:161.09ms