13daf1c4-986b-4938-b269-1ac935891505.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
from dataclasses import dataclass
from pathlib import Path

import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
# Use of FlexAttention contributed by @KoszarskyB
from torch.nn.attention.flex_attention import BlockMask, flex_attention

# -----------------------------------------------------------------------------
# Muon optimizer

def zeropower_via_svd(G, steps=None):
    U, S, V = G.svd()
    return U @ V.T

@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert len(G.shape) == 2
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    X /= (X.norm() + eps) # ensure top singular value <= 1
    if G.size(0) > G.size(1):
        X = X.T
    for _ in range(steps):
        A = X @ X.T
        B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    if G.size(0) > G.size(1):
        X = X.T
    return X

zeropower_backends = dict(svd=zeropower_via_svd, newtonschulz5=zeropower_via_newtonschulz5)

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven't tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        backend: The chosen backend for the orthogonalization step. (recommended: 'newtonschulz5')
        backend_steps: The number of iteration steps to use in the backend, if it is iterative.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True,
                 backend='newtonschulz5', backend_steps=5):
        self.num_process = int(os.environ['WORLD_SIZE'])
        self.rank = int(os.environ["RANK"])
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, backend=backend, backend_steps=backend_steps)
        params: "list[torch.Tensor]" = list(params)
        assert all(isinstance(p, torch.Tensor) for p in params)
        sizes = {p.numel() for p in params}
        param_groups = [
            {
                "params": [p for p in params if p.numel() == size],
                "update_buffer": [
                    torch.empty(size, device="cuda", dtype=torch.bfloat16)
                    for _ in range(self.num_process)
                ],
            }
            for size in sizes
        ]
        super().__init__(param_groups, defaults)

    def step(self):
        for group in self.param_groups:
            lr: float = group["lr"]
            momentum: float = group["momentum"]
            nesterov: bool = group["nesterov"]
            zeropower_backend = zeropower_backends[group["backend"]]
            backend_steps: int = group["backend_steps"]
            update_buffers: "list[torch.Tensor]" = group["update_buffer"]
            # generate weight updates in distributed fashion
            params: "list[torch.Tensor]" = group["params"]
            assert len(params) % self.num_process == 0
            handle = None
            params_world = None
            def update_prev():
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffers):
                    p_world.data.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.num_process]:
                p = params[base_i + self.rank]
                g = p.grad
                assert g is not None
                state = self.state[p] 
                if "momentum_buffer" not in state:
                    state["momentum_buffer"] = torch.zeros_like(g)
                buf: torch.Tensor = state["momentum_buffer"]
                buf.lerp_(g, 1 - momentum)
                g = g.lerp_(buf, momentum) if nesterov else buf
                g = zeropower_backend(g, steps=backend_steps).flatten()
                update_prev()
                handle = dist.all_gather(update_buffers, g, async_op=True)
                params_world = params[base_i : base_i + self.num_process]
            update_prev()


# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):

    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=False)

    def forward(self, x):
        return F.linear(x, self.weight.to(x.dtype))

class Rotary(torch.nn.Module):

    def __init__(self, dim, base=10000):
        super().__init__()
        self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None

    def forward(self, x):
        seq_len = x.shape[1]
        if seq_len != self.seq_len_cached:
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.outer(t, self.inv_freq)
            self.seq_len_cached = seq_len
            self.cos_cached = freqs.cos()
            self.sin_cached = freqs.sin()
        cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
        # apply_rotary_emb(x, cos, sin)
        x1, x2 = x.chunk(2, dim=3)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x)

class CausalSelfAttention(nn.Module):

    def __init__(self, dim, n_head):
        super().__init__()
        assert dim % n_head == 0
        self.n_head = n_head
        self.c_q = CastedLinear(dim, dim)
        self.c_k = CastedLinear(dim, dim)
        self.c_v = CastedLinear(dim, dim)
        # value residual lambda
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5])) # @Grad62304977
        # rotary embeddings
        self.rotary = Rotary(dim // n_head) # dim // n_head = head_dim
        # output projection
        self.c_proj = CastedLinear(dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor, vi: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q: torch.Tensor = self.c_q(x).view(B, T, self.n_head, -1)
        k: torch.Tensor = self.c_k(x).view(B, T, self.n_head, -1)
        v: torch.Tensor = self.c_v(x).view(B, T, self.n_head, -1)
        v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @Grad62304977
        q, k = norm(q), norm(k) # QK norm suggested by @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask)
        y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):

    def __init__(self, dim: int):
        super().__init__()
        self.c_fc   = CastedLinear(dim, 4 * dim)
        self.c_proj = CastedLinear(4 * dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):

    def __init__(self, config):
        super().__init__()
        self.attn = CausalSelfAttention(config.n_embd, config.n_head)
        self.mlp = MLP(config.n_embd)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x: torch.Tensor, vi: torch.Tensor, x0: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        x = x + self.attn(norm(x), vi, block_mask)
        x = x + self.mlp(norm(x))
        return x

# -----------------------------------------------------------------------------
# The main GPT-2 model

@dataclass
class GPTConfig:
    vocab_size : int = 50304
    n_layer : int = 12
    n_head : int = 6 # head dim 128 suggested by @Grad62304977
    n_embd : int = 768
    lm_head_softcap : int = 30

class GPT(nn.Module):

    def __init__(self, config: GPTConfig):
        super().__init__()
        self.n_layer = config.n_layer
        self.lm_head_softcap = config.lm_head_softcap

        # U-net design by @brendanh0gan
        self.num_encoder_layers = config.n_layer // 2 # Half of the layers for encoder
        self.num_decoder_layers = config.n_layer - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))

        self.transformer = nn.ModuleDict(dict(
            wte = nn.Embedding(config.vocab_size, config.n_embd),
            # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
            # U-net structure on token value embeddings by @leloykun
            vte = nn.Embedding(config.vocab_size, config.n_embd*self.num_encoder_layers),
            h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
        ))
        self.lm_head = CastedLinear(config.n_embd, config.vocab_size)
        self.lm_head.weight.data.zero_() # @Grad62304977

    def forward(self, idx: torch.Tensor, target: torch.Tensor, sliding_window: torch.Tensor) -> torch.Tensor:
        BLOCK_SIZE = 128
        assert idx.ndim == 1
        docs = (idx == 50256).cumsum(0)
        docs_low = docs.reshape(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.reshape(-1, BLOCK_SIZE)[:, -1].contiguous()
        def document_sliding_window_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            window_mask = q_idx - kv_idx < sliding_window
            return causal_mask & document_mask & window_mask

        S = len(idx)
        def create_sliding_window_causal_mask(S: int, sliding_window: torch.Tensor):
            kv_idx = block_idx = torch.arange(S // BLOCK_SIZE, dtype=torch.int32, device="cuda")
            q_idx = block_idx[:, None]
            causal_mask = q_idx >= kv_idx
            document_mask = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
            window_mask = q_idx - kv_idx < ((sliding_window + BLOCK_SIZE - 1) // BLOCK_SIZE)
            dense_mask = causal_mask & document_mask & window_mask
            dense_mask = dense_mask.to(torch.int32)
            num_blocks = dense_mask.sum(dim=-1).to(torch.int32)
            indices = torch.argsort(dense_mask, dim=-1, descending=True, stable=True).to(torch.int32)
            num_blocks = num_blocks[None, None, :].contiguous()
            indices = indices[None, None, :].contiguous()
            return BlockMask.from_kv_blocks(num_blocks, indices, BLOCK_SIZE=BLOCK_SIZE, mask_mod=document_sliding_window_causal)
        block_mask = create_sliding_window_causal_mask(S, sliding_window)

        # forward the GPT model itself
        x = self.transformer.wte(idx[None]) # token embeddings of shape (b, t, n_embd)
        x = norm(x) # @Grad62304977
        x0 = x
        vi = self.transformer.vte(idx[None]).chunk(self.num_encoder_layers, dim=-1)

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        for i in range(self.num_encoder_layers):
            x = self.transformer.h[i](x, vi[i], x0, block_mask)
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            # U-net structure on token value embeddings by @leloykun
            x = self.transformer.h[self.num_encoder_layers + i](x, vi[self.num_encoder_layers-1-i], x0, block_mask)

        x = norm(x)
        logits = self.lm_head(x)
        logits = self.lm_head_softcap * torch.tanh(logits / self.lm_head_softcap) # @Grad62304977
        logits = logits.float()
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), target.view(-1))
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _peek_data_shard(file: Path):
    # only reads the header, returns header data
    # header is 256 int32
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    return int(header[2]) # number of tokens (claimed)

def _load_data_shard(file: Path, ntok: int):
    with file.open("rb") as f:
        tokens = torch.empty(ntok, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy())
        assert nbytes == 2 * ntok, "number of tokens read does not match header?"
    return tokens

class DistributedDataLoader:
    def __init__(self, filename_pattern, T, process_rank, num_processes):
        self.process_rank = process_rank
        self.num_processes = num_processes
        self.T = T

        # glob files that match the pattern
        self.files = sorted(Path.cwd().glob(filename_pattern))
        assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"

        # load and validate all data shards, count number of tokens in total
        self.ntoks = [_peek_data_shard(file) for file in self.files]
        assert min(self.ntoks) >= num_processes * T + 1
        self.ntok_total = sum(self.ntoks)

        self.reset()

    def reset(self):
        self.current_shard = -1
        self.advance()

    def advance(self): # advance to next data shard
        self.current_shard = (self.current_shard + 1) % len(self.files)
        self.current_position = self.process_rank * self.T
        self.tokens = _load_data_shard(self.files[self.current_shard], self.ntoks[self.current_shard])

    def next_batch(self):
        batch_size = self.T * self.num_processes
        buf = self.tokens[self.current_position:self.current_position+self.T+1]
        # host side async is sufficient;
        # no performance improvement was observed when introducing a separate stream.
        x = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
        y = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
        # advance current position and load next shard if necessary
        self.current_position += batch_size
        if self.current_position + batch_size + 1 >= len(self.tokens):
            self.advance()
        return x, y

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data hyperparams
    input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
    input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
    # optimization hyperparams
    batch_size : int = 8 # batch size, in sequences, across all devices
    sequence_length : int = 64*1024 # sequence length, in tokens
    num_iterations : int = 1480 # number of iterations to run
    warmup_iters : int = 0
    cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
    weight_decay : float = 0
    # evaluation and logging hyperparams
    val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
    val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()

# set up DDP (distributed data parallel). torchrun sets this env variable
assert torch.cuda.is_available()
dist.init_process_group(backend='nccl')
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
device = f'cuda:{ddp_local_rank}'
torch.cuda.set_device(device)
print(f"using device: {device}")
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = str(uuid.uuid4())
    logdir = 'logs/%s/' % run_id
    # os.makedirs(logdir, exist_ok=True)
    logfile = 'logs/%s.txt' % run_id
    # create the log file
    with open(logfile, "w") as f:
        # begin the log by printing this file (the Python code)
        f.write(code)
        f.write('='*100 + '\n')
def print0(s, logonly=False):
    if master_process:
        with open(logfile, "a") as f:
            if not logonly:
                print(s)
            f.write(s+'\n')
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)

# convenience variables
T = args.sequence_length
# calculate the number of steps to take in the val loop.
assert args.val_tokens % (T * ddp_world_size) == 0
val_steps = args.val_tokens // (T * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size
assert train_accumulation_steps == 1

# load tokens
train_loader = DistributedDataLoader(args.input_bin, T, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, T, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.ntok_total} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.ntok_total} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
x, y = train_loader.next_batch()

# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, n_layer=12, n_head=6, n_embd=768))
model = model.cuda().bfloat16()
for m in model.modules():
    if isinstance(m, CastedLinear):
        m.float()
if hasattr(config, "coordinate_descent_tuning"):
    config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank])
raw_model = model.module # always contains the "raw" unwrapped model

# init the optimizer(s)
optimizer1 = torch.optim.Adam([raw_model.transformer.wte.weight, raw_model.transformer.vte.weight], lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.transformer.h.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
    assert it <= args.num_iterations
    # 1) linear warmup for warmup_iters steps
    if it < args.warmup_iters:
        return (it+1) / args.warmup_iters
    # 2) constant lr for a while
    elif it < args.num_iterations - args.cooldown_iters:
        return 1.0
    # 3) linear cooldown
    else:
        decay_ratio = (args.num_iterations - it) / args.cooldown_iters
        return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]

sliding_window_size = torch.tensor(64, dtype=torch.int32, device="cuda")
sw_size_prev = 64
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
    last_step = (step == args.num_iterations)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Set the sliding window size for the current step, in chunks of 64. By @fernbear.bsky.social
    sw_size =  64 * int((64 + (1792 - 64) * step / args.num_iterations) // 64)
    if sw_size != sw_size_prev:
        sliding_window_size.copy_(sw_size, non_blocking=True)
        sw_size_prev = sw_size

    # once in a while evaluate the validation dataset
    if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # run validation batches
        model.eval()
        val_loader.reset()
        val_loss = 0.0
        for _ in range(val_steps):
            with torch.no_grad():
                x_val, y_val = val_loader.next_batch()
                val_loss += model(x_val, y_val, sliding_window=sliding_window_size)
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        val_loss /= val_steps
        # log val loss to console and to logfile
        print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # save the state of the training process
        log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
        # torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    # bit confusing: we want to make sure to eval on 0th iteration
    # but also after the very last iteration. so we loop for step <= num_iterations
    # instead of just < num_iterations (one extra due to <=), only to do
    # the validation/sampling one last time, and then we break right here as we're done.
    if last_step:
        break

    # --------------- TRAINING SECTION BEGIN -----------------
    model.train()
    loss = model(x, y, sliding_window=sliding_window_size)
    loss.backward()
    del loss
    # advance the dataset for the next batch
    x, y = train_loader.next_batch()
    # momentum warmup for Muon
    frac = min(step/300, 1)
    for group in optimizer3.param_groups:
        group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # --------------- TRAINING SECTION END -------------------
    # everything that follows now is just diagnostics, prints, logging, etc.
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")

if master_process:
    print(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")

# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()
====================================================================================================
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Sun Dec  8 13:45:00 2024       
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.6     |
|-----------------------------------------+----------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
|                                         |                      |               MIG M. |
|=========================================+======================+======================|
|   0  NVIDIA H100 80GB HBM3          On  | 00000000:65:02.0 Off |                    0 |
| N/A   36C    P0              73W / 700W |      7MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  | 00000000:67:02.0 Off |                    0 |
| N/A   45C    P0              98W / 700W |     26MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  | 00000000:69:02.0 Off |                    0 |
| N/A   45C    P0             123W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  | 00000000:6B:02.0 Off |                    0 |
| N/A   39C    P0             118W / 700W |    533MiB / 81559MiB |      1%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  | 00000000:6F:02.0 Off |                    0 |
| N/A   38C    P0              98W / 700W |     26MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  | 00000000:71:02.0 Off |                    0 |
| N/A   45C    P0             100W / 700W |     27MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  | 00000000:73:02.0 Off |                    0 |
| N/A   46C    P0             127W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  | 00000000:75:02.0 Off |                    0 |
| N/A   38C    P0             124W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
                                                                                         
+---------------------------------------------------------------------------------------+
| Processes:                                                                            |
|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
|        ID   ID                                                             Usage      |
|=======================================================================================|
+---------------------------------------------------------------------------------------+

====================================================================================================
Training DataLoader: total number of tokens: 3200000000 across 32 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:24214ms step_avg:nanms
step:2/1480 train_time:24300ms step_avg:nanms
step:3/1480 train_time:24439ms step_avg:nanms
step:4/1480 train_time:24579ms step_avg:nanms
step:5/1480 train_time:24719ms step_avg:nanms
step:6/1480 train_time:24860ms step_avg:nanms
step:7/1480 train_time:25000ms step_avg:nanms
step:8/1480 train_time:25141ms step_avg:nanms
step:9/1480 train_time:25288ms step_avg:nanms
step:10/1480 train_time:25432ms step_avg:nanms
step:11/1480 train_time:142ms step_avg:nanms
step:12/1480 train_time:284ms step_avg:nanms
step:13/1480 train_time:426ms step_avg:141.89ms
step:14/1480 train_time:566ms step_avg:141.61ms
step:15/1480 train_time:708ms step_avg:141.64ms
step:16/1480 train_time:851ms step_avg:141.89ms
step:17/1480 train_time:995ms step_avg:142.19ms
step:18/1480 train_time:1139ms step_avg:142.36ms
step:19/1480 train_time:1282ms step_avg:142.43ms
step:20/1480 train_time:1424ms step_avg:142.36ms
step:21/1480 train_time:1565ms step_avg:142.31ms
step:22/1480 train_time:1706ms step_avg:142.19ms
step:23/1480 train_time:1850ms step_avg:142.33ms
step:24/1480 train_time:1993ms step_avg:142.34ms
step:25/1480 train_time:2138ms step_avg:142.53ms
step:26/1480 train_time:2281ms step_avg:142.57ms
step:27/1480 train_time:2424ms step_avg:142.58ms
step:28/1480 train_time:2566ms step_avg:142.54ms
step:29/1480 train_time:2707ms step_avg:142.48ms
step:30/1480 train_time:2850ms step_avg:142.51ms
step:31/1480 train_time:2991ms step_avg:142.41ms
step:32/1480 train_time:3135ms step_avg:142.49ms
step:33/1480 train_time:3279ms step_avg:142.55ms
step:34/1480 train_time:3422ms step_avg:142.57ms
step:35/1480 train_time:3565ms step_avg:142.60ms
step:36/1480 train_time:3707ms step_avg:142.56ms
step:37/1480 train_time:3849ms step_avg:142.55ms
step:38/1480 train_time:3990ms step_avg:142.52ms
step:39/1480 train_time:4134ms step_avg:142.55ms
step:40/1480 train_time:4278ms step_avg:142.59ms
step:41/1480 train_time:4420ms step_avg:142.58ms
step:42/1480 train_time:4563ms step_avg:142.59ms
step:43/1480 train_time:4705ms step_avg:142.58ms
step:44/1480 train_time:4848ms step_avg:142.59ms
step:45/1480 train_time:4990ms step_avg:142.56ms
step:46/1480 train_time:5134ms step_avg:142.62ms
step:47/1480 train_time:5278ms step_avg:142.66ms
step:48/1480 train_time:5422ms step_avg:142.69ms
step:49/1480 train_time:5566ms step_avg:142.72ms
step:50/1480 train_time:5707ms step_avg:142.67ms
step:51/1480 train_time:5848ms step_avg:142.64ms
step:52/1480 train_time:5990ms step_avg:142.62ms
step:53/1480 train_time:6133ms step_avg:142.63ms
step:54/1480 train_time:6278ms step_avg:142.69ms
step:55/1480 train_time:6422ms step_avg:142.71ms
step:56/1480 train_time:6565ms step_avg:142.72ms
step:57/1480 train_time:6707ms step_avg:142.70ms
step:58/1480 train_time:6848ms step_avg:142.67ms
step:59/1480 train_time:6989ms step_avg:142.63ms
step:60/1480 train_time:7133ms step_avg:142.66ms
step:61/1480 train_time:7278ms step_avg:142.71ms
step:62/1480 train_time:7422ms step_avg:142.73ms
step:63/1480 train_time:7565ms step_avg:142.74ms
step:64/1480 train_time:7707ms step_avg:142.72ms
step:65/1480 train_time:7848ms step_avg:142.69ms
step:66/1480 train_time:7988ms step_avg:142.65ms
step:67/1480 train_time:8130ms step_avg:142.63ms
step:68/1480 train_time:8274ms step_avg:142.65ms
step:69/1480 train_time:8417ms step_avg:142.67ms
step:70/1480 train_time:8561ms step_avg:142.69ms
step:71/1480 train_time:8703ms step_avg:142.68ms
step:72/1480 train_time:8845ms step_avg:142.67ms
step:73/1480 train_time:8987ms step_avg:142.64ms
step:74/1480 train_time:9130ms step_avg:142.66ms
step:75/1480 train_time:9274ms step_avg:142.68ms
step:76/1480 train_time:9416ms step_avg:142.67ms
step:77/1480 train_time:9564ms step_avg:142.75ms
step:78/1480 train_time:9707ms step_avg:142.74ms
step:79/1480 train_time:9848ms step_avg:142.72ms
step:80/1480 train_time:9988ms step_avg:142.69ms
step:81/1480 train_time:10130ms step_avg:142.67ms
step:82/1480 train_time:10271ms step_avg:142.66ms
step:83/1480 train_time:10415ms step_avg:142.67ms
step:84/1480 train_time:10559ms step_avg:142.70ms
step:85/1480 train_time:10704ms step_avg:142.72ms
step:86/1480 train_time:10846ms step_avg:142.71ms
step:87/1480 train_time:10987ms step_avg:142.69ms
step:88/1480 train_time:11128ms step_avg:142.67ms
step:89/1480 train_time:11271ms step_avg:142.66ms
step:90/1480 train_time:11413ms step_avg:142.67ms
step:91/1480 train_time:11557ms step_avg:142.68ms
step:92/1480 train_time:11701ms step_avg:142.69ms
step:93/1480 train_time:11844ms step_avg:142.69ms
step:94/1480 train_time:11985ms step_avg:142.68ms
step:95/1480 train_time:12127ms step_avg:142.67ms
step:96/1480 train_time:12269ms step_avg:142.66ms
step:97/1480 train_time:12411ms step_avg:142.66ms
step:98/1480 train_time:12556ms step_avg:142.68ms
step:99/1480 train_time:12700ms step_avg:142.70ms
step:100/1480 train_time:12844ms step_avg:142.71ms
step:101/1480 train_time:12986ms step_avg:142.70ms
step:102/1480 train_time:13127ms step_avg:142.69ms
step:103/1480 train_time:13270ms step_avg:142.69ms
step:104/1480 train_time:13413ms step_avg:142.70ms
step:105/1480 train_time:13557ms step_avg:142.71ms
step:106/1480 train_time:13701ms step_avg:142.72ms
step:107/1480 train_time:13843ms step_avg:142.71ms
step:108/1480 train_time:13984ms step_avg:142.70ms
step:109/1480 train_time:14125ms step_avg:142.68ms
step:110/1480 train_time:14268ms step_avg:142.68ms
step:111/1480 train_time:14413ms step_avg:142.70ms
step:112/1480 train_time:14562ms step_avg:142.76ms
step:113/1480 train_time:14709ms step_avg:142.81ms
step:114/1480 train_time:14858ms step_avg:142.87ms
step:115/1480 train_time:15005ms step_avg:142.91ms
step:116/1480 train_time:15150ms step_avg:142.93ms
step:117/1480 train_time:15296ms step_avg:142.96ms
step:118/1480 train_time:15444ms step_avg:143.00ms
step:119/1480 train_time:15591ms step_avg:143.03ms
step:120/1480 train_time:15739ms step_avg:143.09ms
step:121/1480 train_time:15887ms step_avg:143.13ms
step:122/1480 train_time:16033ms step_avg:143.15ms
step:123/1480 train_time:16180ms step_avg:143.18ms
step:124/1480 train_time:16325ms step_avg:143.21ms
step:125/1480 train_time:16472ms step_avg:143.24ms
step:125/1480 val_loss:4.4101 train_time:16529ms step_avg:143.73ms
step:126/1480 train_time:16623ms step_avg:143.30ms
step:127/1480 train_time:16772ms step_avg:143.35ms
step:128/1480 train_time:16918ms step_avg:143.37ms
step:129/1480 train_time:17062ms step_avg:143.38ms
step:130/1480 train_time:17209ms step_avg:143.41ms
step:131/1480 train_time:17355ms step_avg:143.43ms
step:132/1480 train_time:17501ms step_avg:143.45ms
step:133/1480 train_time:17650ms step_avg:143.50ms
step:134/1480 train_time:17799ms step_avg:143.54ms
step:135/1480 train_time:17944ms step_avg:143.56ms
step:136/1480 train_time:18092ms step_avg:143.59ms
step:137/1480 train_time:18238ms step_avg:143.60ms
step:138/1480 train_time:18383ms step_avg:143.62ms
step:139/1480 train_time:18530ms step_avg:143.64ms
step:140/1480 train_time:18678ms step_avg:143.68ms
step:141/1480 train_time:18824ms step_avg:143.70ms
step:142/1480 train_time:18972ms step_avg:143.73ms
step:143/1480 train_time:19120ms step_avg:143.76ms
step:144/1480 train_time:19265ms step_avg:143.77ms
step:145/1480 train_time:19412ms step_avg:143.79ms
step:146/1480 train_time:19558ms step_avg:143.81ms
step:147/1480 train_time:19706ms step_avg:143.84ms
step:148/1480 train_time:19854ms step_avg:143.87ms
step:149/1480 train_time:20000ms step_avg:143.89ms
step:150/1480 train_time:20146ms step_avg:143.90ms
step:151/1480 train_time:20293ms step_avg:143.93ms
step:152/1480 train_time:20439ms step_avg:143.94ms
step:153/1480 train_time:20587ms step_avg:143.96ms
step:154/1480 train_time:20734ms step_avg:143.98ms
step:155/1480 train_time:20881ms step_avg:144.01ms
step:156/1480 train_time:21029ms step_avg:144.04ms
step:157/1480 train_time:21176ms step_avg:144.06ms
step:158/1480 train_time:21322ms step_avg:144.07ms
step:159/1480 train_time:21468ms step_avg:144.08ms
step:160/1480 train_time:21616ms step_avg:144.10ms
step:161/1480 train_time:21762ms step_avg:144.12ms
step:162/1480 train_time:21910ms step_avg:144.14ms
step:163/1480 train_time:22056ms step_avg:144.16ms
step:164/1480 train_time:22203ms step_avg:144.18ms
step:165/1480 train_time:22350ms step_avg:144.20ms
step:166/1480 train_time:22497ms step_avg:144.21ms
step:167/1480 train_time:22643ms step_avg:144.22ms
step:168/1480 train_time:22791ms step_avg:144.25ms
step:169/1480 train_time:22938ms step_avg:144.27ms
step:170/1480 train_time:23084ms step_avg:144.27ms
step:171/1480 train_time:23231ms step_avg:144.29ms
step:172/1480 train_time:23378ms step_avg:144.31ms
step:173/1480 train_time:23524ms step_avg:144.32ms
step:174/1480 train_time:23671ms step_avg:144.34ms
step:175/1480 train_time:23818ms step_avg:144.35ms
step:176/1480 train_time:23965ms step_avg:144.37ms
step:177/1480 train_time:24114ms step_avg:144.39ms
step:178/1480 train_time:24260ms step_avg:144.40ms
step:179/1480 train_time:24406ms step_avg:144.41ms
step:180/1480 train_time:24552ms step_avg:144.42ms
step:181/1480 train_time:24700ms step_avg:144.45ms
step:182/1480 train_time:24847ms step_avg:144.46ms
step:183/1480 train_time:24994ms step_avg:144.47ms
step:184/1480 train_time:25140ms step_avg:144.49ms
step:185/1480 train_time:25286ms step_avg:144.49ms
step:186/1480 train_time:25433ms step_avg:144.51ms
step:187/1480 train_time:25580ms step_avg:144.52ms
step:188/1480 train_time:25726ms step_avg:144.53ms
step:189/1480 train_time:25873ms step_avg:144.54ms
step:190/1480 train_time:26019ms step_avg:144.55ms
step:191/1480 train_time:26165ms step_avg:144.56ms
step:192/1480 train_time:26314ms step_avg:144.58ms
step:193/1480 train_time:26460ms step_avg:144.59ms
step:194/1480 train_time:26607ms step_avg:144.60ms
step:195/1480 train_time:26754ms step_avg:144.61ms
step:196/1480 train_time:26901ms step_avg:144.63ms
step:197/1480 train_time:27048ms step_avg:144.64ms
step:198/1480 train_time:27195ms step_avg:144.66ms
step:199/1480 train_time:27341ms step_avg:144.66ms
step:200/1480 train_time:27489ms step_avg:144.68ms
step:201/1480 train_time:27635ms step_avg:144.69ms
step:202/1480 train_time:27782ms step_avg:144.70ms
step:203/1480 train_time:27928ms step_avg:144.71ms
step:204/1480 train_time:28076ms step_avg:144.72ms
step:205/1480 train_time:28222ms step_avg:144.73ms
step:206/1480 train_time:28370ms step_avg:144.74ms
step:207/1480 train_time:28517ms step_avg:144.76ms
step:208/1480 train_time:28663ms step_avg:144.76ms
step:209/1480 train_time:28812ms step_avg:144.78ms
step:210/1480 train_time:28958ms step_avg:144.79ms
step:211/1480 train_time:29104ms step_avg:144.79ms
step:212/1480 train_time:29251ms step_avg:144.81ms
step:213/1480 train_time:29398ms step_avg:144.82ms
step:214/1480 train_time:29544ms step_avg:144.82ms
step:215/1480 train_time:29692ms step_avg:144.84ms
step:216/1480 train_time:29840ms step_avg:144.85ms
step:217/1480 train_time:29987ms step_avg:144.87ms
step:218/1480 train_time:30134ms step_avg:144.88ms
step:219/1480 train_time:30281ms step_avg:144.89ms
step:220/1480 train_time:30428ms step_avg:144.89ms
step:221/1480 train_time:30577ms step_avg:144.91ms
step:222/1480 train_time:30728ms step_avg:144.94ms
step:223/1480 train_time:30879ms step_avg:144.97ms
step:224/1480 train_time:31029ms step_avg:145.00ms
step:225/1480 train_time:31180ms step_avg:145.02ms
step:226/1480 train_time:31330ms step_avg:145.04ms
step:227/1480 train_time:31479ms step_avg:145.07ms
step:228/1480 train_time:31629ms step_avg:145.09ms
step:229/1480 train_time:31780ms step_avg:145.12ms
step:230/1480 train_time:31931ms step_avg:145.14ms
step:231/1480 train_time:32082ms step_avg:145.17ms
step:232/1480 train_time:32234ms step_avg:145.20ms
step:233/1480 train_time:32383ms step_avg:145.22ms
step:234/1480 train_time:32534ms step_avg:145.24ms
step:235/1480 train_time:32686ms step_avg:145.27ms
step:236/1480 train_time:32838ms step_avg:145.30ms
step:237/1480 train_time:32987ms step_avg:145.32ms
step:238/1480 train_time:33137ms step_avg:145.34ms
step:239/1480 train_time:33288ms step_avg:145.36ms
step:240/1480 train_time:33438ms step_avg:145.38ms
step:241/1480 train_time:33588ms step_avg:145.40ms
step:242/1480 train_time:33738ms step_avg:145.42ms
step:243/1480 train_time:33890ms step_avg:145.45ms
step:244/1480 train_time:34040ms step_avg:145.47ms
step:245/1480 train_time:34192ms step_avg:145.50ms
step:246/1480 train_time:34342ms step_avg:145.52ms
step:247/1480 train_time:34492ms step_avg:145.54ms
step:248/1480 train_time:34642ms step_avg:145.55ms
step:249/1480 train_time:34793ms step_avg:145.58ms
step:250/1480 train_time:34942ms step_avg:145.59ms
step:250/1480 val_loss:3.9840 train_time:35000ms step_avg:145.83ms
step:251/1480 train_time:35096ms step_avg:145.63ms
step:252/1480 train_time:35248ms step_avg:145.65ms
step:253/1480 train_time:35397ms step_avg:145.67ms
step:254/1480 train_time:35546ms step_avg:145.68ms
step:255/1480 train_time:35696ms step_avg:145.70ms
step:256/1480 train_time:35845ms step_avg:145.71ms
step:257/1480 train_time:35995ms step_avg:145.73ms
step:258/1480 train_time:36146ms step_avg:145.75ms
step:259/1480 train_time:36297ms step_avg:145.77ms
step:260/1480 train_time:36448ms step_avg:145.79ms
step:261/1480 train_time:36598ms step_avg:145.81ms
step:262/1480 train_time:36749ms step_avg:145.83ms
step:263/1480 train_time:36898ms step_avg:145.84ms
step:264/1480 train_time:37049ms step_avg:145.86ms
step:265/1480 train_time:37199ms step_avg:145.88ms
step:266/1480 train_time:37350ms step_avg:145.90ms
step:267/1480 train_time:37501ms step_avg:145.92ms
step:268/1480 train_time:37651ms step_avg:145.93ms
step:269/1480 train_time:37801ms step_avg:145.95ms
step:270/1480 train_time:37951ms step_avg:145.97ms
step:271/1480 train_time:38102ms step_avg:145.99ms
step:272/1480 train_time:38253ms step_avg:146.01ms
step:273/1480 train_time:38405ms step_avg:146.03ms
step:274/1480 train_time:38555ms step_avg:146.04ms
step:275/1480 train_time:38706ms step_avg:146.06ms
step:276/1480 train_time:38856ms step_avg:146.08ms
step:277/1480 train_time:39005ms step_avg:146.09ms
step:278/1480 train_time:39155ms step_avg:146.10ms
step:279/1480 train_time:39305ms step_avg:146.12ms
step:280/1480 train_time:39456ms step_avg:146.13ms
step:281/1480 train_time:39607ms step_avg:146.15ms
step:282/1480 train_time:39757ms step_avg:146.17ms
step:283/1480 train_time:39909ms step_avg:146.19ms
step:284/1480 train_time:40059ms step_avg:146.20ms
step:285/1480 train_time:40209ms step_avg:146.22ms
step:286/1480 train_time:40360ms step_avg:146.23ms
step:287/1480 train_time:40511ms step_avg:146.25ms
step:288/1480 train_time:40661ms step_avg:146.26ms
step:289/1480 train_time:40812ms step_avg:146.28ms
step:290/1480 train_time:40961ms step_avg:146.29ms
step:291/1480 train_time:41112ms step_avg:146.31ms
step:292/1480 train_time:41262ms step_avg:146.32ms
step:293/1480 train_time:41413ms step_avg:146.34ms
step:294/1480 train_time:41564ms step_avg:146.35ms
step:295/1480 train_time:41715ms step_avg:146.37ms
step:296/1480 train_time:41866ms step_avg:146.39ms
step:297/1480 train_time:42018ms step_avg:146.40ms
step:298/1480 train_time:42169ms step_avg:146.42ms
step:299/1480 train_time:42320ms step_avg:146.44ms
step:300/1480 train_time:42471ms step_avg:146.45ms
step:301/1480 train_time:42622ms step_avg:146.47ms
step:302/1480 train_time:42772ms step_avg:146.48ms
step:303/1480 train_time:42923ms step_avg:146.49ms
step:304/1480 train_time:43073ms step_avg:146.51ms
step:305/1480 train_time:43224ms step_avg:146.52ms
step:306/1480 train_time:43375ms step_avg:146.54ms
step:307/1480 train_time:43526ms step_avg:146.55ms
step:308/1480 train_time:43677ms step_avg:146.57ms
step:309/1480 train_time:43830ms step_avg:146.59ms
step:310/1480 train_time:43979ms step_avg:146.60ms
step:311/1480 train_time:44130ms step_avg:146.61ms
step:312/1480 train_time:44280ms step_avg:146.62ms
step:313/1480 train_time:44432ms step_avg:146.64ms
step:314/1480 train_time:44583ms step_avg:146.65ms
step:315/1480 train_time:44733ms step_avg:146.67ms
step:316/1480 train_time:44883ms step_avg:146.68ms
step:317/1480 train_time:45034ms step_avg:146.69ms
step:318/1480 train_time:45183ms step_avg:146.70ms
step:319/1480 train_time:45334ms step_avg:146.71ms
step:320/1480 train_time:45486ms step_avg:146.73ms
step:321/1480 train_time:45636ms step_avg:146.74ms
step:322/1480 train_time:45787ms step_avg:146.75ms
step:323/1480 train_time:45937ms step_avg:146.76ms
step:324/1480 train_time:46089ms step_avg:146.78ms
step:325/1480 train_time:46238ms step_avg:146.79ms
step:326/1480 train_time:46389ms step_avg:146.80ms
step:327/1480 train_time:46539ms step_avg:146.81ms
step:328/1480 train_time:46691ms step_avg:146.83ms
step:329/1480 train_time:46841ms step_avg:146.84ms
step:330/1480 train_time:46994ms step_avg:146.86ms
step:331/1480 train_time:47151ms step_avg:146.89ms
step:332/1480 train_time:47303ms step_avg:146.90ms
step:333/1480 train_time:47456ms step_avg:146.92ms
step:334/1480 train_time:47610ms step_avg:146.94ms
step:335/1480 train_time:47764ms step_avg:146.97ms
step:336/1480 train_time:47917ms step_avg:146.98ms
step:337/1480 train_time:48070ms step_avg:147.00ms
step:338/1480 train_time:48224ms step_avg:147.03ms
step:339/1480 train_time:48377ms step_avg:147.04ms
step:340/1480 train_time:48531ms step_avg:147.06ms
step:341/1480 train_time:48684ms step_avg:147.08ms
step:342/1480 train_time:48837ms step_avg:147.10ms
step:343/1480 train_time:48992ms step_avg:147.12ms
step:344/1480 train_time:49147ms step_avg:147.15ms
step:345/1480 train_time:49302ms step_avg:147.17ms
step:346/1480 train_time:49456ms step_avg:147.19ms
step:347/1480 train_time:49611ms step_avg:147.21ms
step:348/1480 train_time:49765ms step_avg:147.24ms
step:349/1480 train_time:49919ms step_avg:147.25ms
step:350/1480 train_time:50073ms step_avg:147.27ms
step:351/1480 train_time:50229ms step_avg:147.30ms
step:352/1480 train_time:50383ms step_avg:147.32ms
step:353/1480 train_time:50536ms step_avg:147.34ms
step:354/1480 train_time:50690ms step_avg:147.36ms
step:355/1480 train_time:50845ms step_avg:147.38ms
step:356/1480 train_time:50999ms step_avg:147.40ms
step:357/1480 train_time:51153ms step_avg:147.42ms
step:358/1480 train_time:51309ms step_avg:147.44ms
step:359/1480 train_time:51464ms step_avg:147.46ms
step:360/1480 train_time:51618ms step_avg:147.48ms
step:361/1480 train_time:51772ms step_avg:147.50ms
step:362/1480 train_time:51929ms step_avg:147.53ms
step:363/1480 train_time:52083ms step_avg:147.54ms
step:364/1480 train_time:52237ms step_avg:147.56ms
step:365/1480 train_time:52391ms step_avg:147.58ms
step:366/1480 train_time:52545ms step_avg:147.60ms
step:367/1480 train_time:52699ms step_avg:147.61ms
step:368/1480 train_time:52852ms step_avg:147.63ms
step:369/1480 train_time:53007ms step_avg:147.65ms
step:370/1480 train_time:53160ms step_avg:147.67ms
step:371/1480 train_time:53313ms step_avg:147.68ms
step:372/1480 train_time:53467ms step_avg:147.70ms
step:373/1480 train_time:53620ms step_avg:147.71ms
step:374/1480 train_time:53773ms step_avg:147.73ms
step:375/1480 train_time:53929ms step_avg:147.75ms
step:375/1480 val_loss:3.8024 train_time:53990ms step_avg:147.92ms
step:376/1480 train_time:54088ms step_avg:147.78ms
step:377/1480 train_time:54243ms step_avg:147.80ms
step:378/1480 train_time:54395ms step_avg:147.81ms
step:379/1480 train_time:54547ms step_avg:147.83ms
step:380/1480 train_time:54699ms step_avg:147.84ms
step:381/1480 train_time:54852ms step_avg:147.85ms
step:382/1480 train_time:55007ms step_avg:147.87ms
step:383/1480 train_time:55163ms step_avg:147.89ms
step:384/1480 train_time:55316ms step_avg:147.90ms
step:385/1480 train_time:55469ms step_avg:147.92ms
step:386/1480 train_time:55624ms step_avg:147.94ms
step:387/1480 train_time:55777ms step_avg:147.95ms
step:388/1480 train_time:55931ms step_avg:147.97ms
step:389/1480 train_time:56083ms step_avg:147.98ms
step:390/1480 train_time:56237ms step_avg:147.99ms
step:391/1480 train_time:56391ms step_avg:148.01ms
step:392/1480 train_time:56545ms step_avg:148.02ms
step:393/1480 train_time:56699ms step_avg:148.04ms
step:394/1480 train_time:56852ms step_avg:148.05ms
step:395/1480 train_time:57007ms step_avg:148.07ms
step:396/1480 train_time:57160ms step_avg:148.08ms
step:397/1480 train_time:57315ms step_avg:148.10ms
step:398/1480 train_time:57469ms step_avg:148.12ms
step:399/1480 train_time:57625ms step_avg:148.14ms
step:400/1480 train_time:57779ms step_avg:148.15ms
step:401/1480 train_time:57933ms step_avg:148.17ms
step:402/1480 train_time:58086ms step_avg:148.18ms
step:403/1480 train_time:58240ms step_avg:148.19ms
step:404/1480 train_time:58394ms step_avg:148.21ms
step:405/1480 train_time:58548ms step_avg:148.22ms
step:406/1480 train_time:58702ms step_avg:148.24ms
step:407/1480 train_time:58856ms step_avg:148.25ms
step:408/1480 train_time:59009ms step_avg:148.26ms
step:409/1480 train_time:59162ms step_avg:148.28ms
step:410/1480 train_time:59315ms step_avg:148.29ms
step:411/1480 train_time:59468ms step_avg:148.30ms
step:412/1480 train_time:59624ms step_avg:148.32ms
step:413/1480 train_time:59777ms step_avg:148.33ms
step:414/1480 train_time:59932ms step_avg:148.35ms
step:415/1480 train_time:60086ms step_avg:148.36ms
step:416/1480 train_time:60239ms step_avg:148.37ms
step:417/1480 train_time:60393ms step_avg:148.39ms
step:418/1480 train_time:60546ms step_avg:148.40ms
step:419/1480 train_time:60700ms step_avg:148.41ms
step:420/1480 train_time:60853ms step_avg:148.42ms
step:421/1480 train_time:61007ms step_avg:148.44ms
step:422/1480 train_time:61160ms step_avg:148.45ms
step:423/1480 train_time:61313ms step_avg:148.46ms
step:424/1480 train_time:61468ms step_avg:148.47ms
step:425/1480 train_time:61622ms step_avg:148.49ms
step:426/1480 train_time:61776ms step_avg:148.50ms
step:427/1480 train_time:61929ms step_avg:148.51ms
step:428/1480 train_time:62082ms step_avg:148.52ms
step:429/1480 train_time:62235ms step_avg:148.53ms
step:430/1480 train_time:62389ms step_avg:148.54ms
step:431/1480 train_time:62543ms step_avg:148.56ms
step:432/1480 train_time:62697ms step_avg:148.57ms
step:433/1480 train_time:62851ms step_avg:148.58ms
step:434/1480 train_time:63004ms step_avg:148.59ms
step:435/1480 train_time:63157ms step_avg:148.60ms
step:436/1480 train_time:63311ms step_avg:148.62ms
step:437/1480 train_time:63464ms step_avg:148.63ms
step:438/1480 train_time:63618ms step_avg:148.64ms
step:439/1480 train_time:63771ms step_avg:148.65ms
step:440/1480 train_time:63928ms step_avg:148.67ms
step:441/1480 train_time:64085ms step_avg:148.69ms
step:442/1480 train_time:64243ms step_avg:148.71ms
step:443/1480 train_time:64399ms step_avg:148.73ms
step:444/1480 train_time:64555ms step_avg:148.74ms
step:445/1480 train_time:64710ms step_avg:148.76ms
step:446/1480 train_time:64867ms step_avg:148.78ms
step:447/1480 train_time:65024ms step_avg:148.80ms
step:448/1480 train_time:65180ms step_avg:148.81ms
step:449/1480 train_time:65336ms step_avg:148.83ms
step:450/1480 train_time:65494ms step_avg:148.85ms
step:451/1480 train_time:65651ms step_avg:148.87ms
step:452/1480 train_time:65809ms step_avg:148.89ms
step:453/1480 train_time:65964ms step_avg:148.90ms
step:454/1480 train_time:66121ms step_avg:148.92ms
step:455/1480 train_time:66277ms step_avg:148.94ms
step:456/1480 train_time:66433ms step_avg:148.95ms
step:457/1480 train_time:66590ms step_avg:148.97ms
step:458/1480 train_time:66747ms step_avg:148.99ms
step:459/1480 train_time:66904ms step_avg:149.01ms
step:460/1480 train_time:67060ms step_avg:149.02ms
step:461/1480 train_time:67220ms step_avg:149.05ms
step:462/1480 train_time:67375ms step_avg:149.06ms
step:463/1480 train_time:67531ms step_avg:149.08ms
step:464/1480 train_time:67688ms step_avg:149.09ms
step:465/1480 train_time:67845ms step_avg:149.11ms
step:466/1480 train_time:68001ms step_avg:149.12ms
step:467/1480 train_time:68158ms step_avg:149.14ms
step:468/1480 train_time:68313ms step_avg:149.16ms
step:469/1480 train_time:68470ms step_avg:149.17ms
step:470/1480 train_time:68629ms step_avg:149.19ms
step:471/1480 train_time:68786ms step_avg:149.21ms
step:472/1480 train_time:68944ms step_avg:149.23ms
step:473/1480 train_time:69100ms step_avg:149.24ms
step:474/1480 train_time:69256ms step_avg:149.26ms
step:475/1480 train_time:69411ms step_avg:149.27ms
step:476/1480 train_time:69568ms step_avg:149.29ms
step:477/1480 train_time:69726ms step_avg:149.31ms
step:478/1480 train_time:69882ms step_avg:149.32ms
step:479/1480 train_time:70038ms step_avg:149.34ms
step:480/1480 train_time:70196ms step_avg:149.35ms
step:481/1480 train_time:70352ms step_avg:149.37ms
step:482/1480 train_time:70509ms step_avg:149.38ms
step:483/1480 train_time:70668ms step_avg:149.40ms
step:484/1480 train_time:70825ms step_avg:149.42ms
step:485/1480 train_time:70983ms step_avg:149.44ms
step:486/1480 train_time:71139ms step_avg:149.45ms
step:487/1480 train_time:71295ms step_avg:149.47ms
step:488/1480 train_time:71452ms step_avg:149.48ms
step:489/1480 train_time:71609ms step_avg:149.50ms
step:490/1480 train_time:71765ms step_avg:149.51ms
step:491/1480 train_time:71922ms step_avg:149.53ms
step:492/1480 train_time:72077ms step_avg:149.54ms
step:493/1480 train_time:72234ms step_avg:149.55ms
step:494/1480 train_time:72391ms step_avg:149.57ms
step:495/1480 train_time:72548ms step_avg:149.58ms
step:496/1480 train_time:72706ms step_avg:149.60ms
step:497/1480 train_time:72863ms step_avg:149.62ms
step:498/1480 train_time:73020ms step_avg:149.63ms
step:499/1480 train_time:73175ms step_avg:149.64ms
step:500/1480 train_time:73333ms step_avg:149.66ms
step:500/1480 val_loss:3.6823 train_time:73394ms step_avg:149.78ms
step:501/1480 train_time:73493ms step_avg:149.68ms
step:502/1480 train_time:73650ms step_avg:149.70ms
step:503/1480 train_time:73808ms step_avg:149.71ms
step:504/1480 train_time:73965ms step_avg:149.73ms
step:505/1480 train_time:74119ms step_avg:149.74ms
step:506/1480 train_time:74275ms step_avg:149.75ms
step:507/1480 train_time:74433ms step_avg:149.76ms
step:508/1480 train_time:74592ms step_avg:149.78ms
step:509/1480 train_time:74748ms step_avg:149.80ms
step:510/1480 train_time:74907ms step_avg:149.81ms
step:511/1480 train_time:75064ms step_avg:149.83ms
step:512/1480 train_time:75221ms step_avg:149.84ms
step:513/1480 train_time:75377ms step_avg:149.85ms
step:514/1480 train_time:75534ms step_avg:149.87ms
step:515/1480 train_time:75692ms step_avg:149.88ms
step:516/1480 train_time:75850ms step_avg:149.90ms
step:517/1480 train_time:76009ms step_avg:149.92ms
step:518/1480 train_time:76167ms step_avg:149.93ms
step:519/1480 train_time:76323ms step_avg:149.95ms
step:520/1480 train_time:76481ms step_avg:149.96ms
step:521/1480 train_time:76637ms step_avg:149.98ms
step:522/1480 train_time:76794ms step_avg:149.99ms
step:523/1480 train_time:76951ms step_avg:150.00ms
step:524/1480 train_time:77109ms step_avg:150.02ms
step:525/1480 train_time:77267ms step_avg:150.03ms
step:526/1480 train_time:77426ms step_avg:150.05ms
step:527/1480 train_time:77585ms step_avg:150.07ms
step:528/1480 train_time:77742ms step_avg:150.08ms
step:529/1480 train_time:77899ms step_avg:150.09ms
step:530/1480 train_time:78055ms step_avg:150.11ms
step:531/1480 train_time:78211ms step_avg:150.12ms
step:532/1480 train_time:78367ms step_avg:150.13ms
step:533/1480 train_time:78523ms step_avg:150.14ms
step:534/1480 train_time:78679ms step_avg:150.15ms
step:535/1480 train_time:78835ms step_avg:150.16ms
step:536/1480 train_time:78993ms step_avg:150.18ms
step:537/1480 train_time:79149ms step_avg:150.19ms
step:538/1480 train_time:79308ms step_avg:150.20ms
step:539/1480 train_time:79468ms step_avg:150.22ms
step:540/1480 train_time:79626ms step_avg:150.24ms
step:541/1480 train_time:79784ms step_avg:150.25ms
step:542/1480 train_time:79941ms step_avg:150.26ms
step:543/1480 train_time:80096ms step_avg:150.27ms
step:544/1480 train_time:80251ms step_avg:150.28ms
step:545/1480 train_time:80409ms step_avg:150.30ms
step:546/1480 train_time:80565ms step_avg:150.31ms
step:547/1480 train_time:80722ms step_avg:150.32ms
step:548/1480 train_time:80881ms step_avg:150.34ms
step:549/1480 train_time:81037ms step_avg:150.35ms
step:550/1480 train_time:81195ms step_avg:150.36ms
step:551/1480 train_time:81353ms step_avg:150.38ms
step:552/1480 train_time:81512ms step_avg:150.39ms
step:553/1480 train_time:81671ms step_avg:150.41ms
step:554/1480 train_time:81831ms step_avg:150.42ms
step:555/1480 train_time:81992ms step_avg:150.44ms
step:556/1480 train_time:82150ms step_avg:150.46ms
step:557/1480 train_time:82311ms step_avg:150.48ms
step:558/1480 train_time:82470ms step_avg:150.49ms
step:559/1480 train_time:82630ms step_avg:150.51ms
step:560/1480 train_time:82790ms step_avg:150.53ms
step:561/1480 train_time:82949ms step_avg:150.54ms
step:562/1480 train_time:83110ms step_avg:150.56ms
step:563/1480 train_time:83269ms step_avg:150.58ms
step:564/1480 train_time:83429ms step_avg:150.59ms
step:565/1480 train_time:83588ms step_avg:150.61ms
step:566/1480 train_time:83749ms step_avg:150.63ms
step:567/1480 train_time:83909ms step_avg:150.64ms
step:568/1480 train_time:84068ms step_avg:150.66ms
step:569/1480 train_time:84228ms step_avg:150.68ms
step:570/1480 train_time:84387ms step_avg:150.69ms
step:571/1480 train_time:84547ms step_avg:150.71ms
step:572/1480 train_time:84708ms step_avg:150.73ms
step:573/1480 train_time:84868ms step_avg:150.74ms
step:574/1480 train_time:85029ms step_avg:150.76ms
step:575/1480 train_time:85191ms step_avg:150.78ms
step:576/1480 train_time:85349ms step_avg:150.79ms
step:577/1480 train_time:85509ms step_avg:150.81ms
step:578/1480 train_time:85669ms step_avg:150.83ms
step:579/1480 train_time:85828ms step_avg:150.84ms
step:580/1480 train_time:85987ms step_avg:150.85ms
step:581/1480 train_time:86148ms step_avg:150.87ms
step:582/1480 train_time:86309ms step_avg:150.89ms
step:583/1480 train_time:86468ms step_avg:150.90ms
step:584/1480 train_time:86628ms step_avg:150.92ms
step:585/1480 train_time:86788ms step_avg:150.94ms
step:586/1480 train_time:86948ms step_avg:150.95ms
step:587/1480 train_time:87108ms step_avg:150.97ms
step:588/1480 train_time:87268ms step_avg:150.98ms
step:589/1480 train_time:87429ms step_avg:151.00ms
step:590/1480 train_time:87590ms step_avg:151.02ms
step:591/1480 train_time:87748ms step_avg:151.03ms
step:592/1480 train_time:87909ms step_avg:151.05ms
step:593/1480 train_time:88070ms step_avg:151.06ms
step:594/1480 train_time:88232ms step_avg:151.08ms
step:595/1480 train_time:88393ms step_avg:151.10ms
step:596/1480 train_time:88553ms step_avg:151.11ms
step:597/1480 train_time:88712ms step_avg:151.13ms
step:598/1480 train_time:88870ms step_avg:151.14ms
step:599/1480 train_time:89029ms step_avg:151.15ms
step:600/1480 train_time:89189ms step_avg:151.17ms
step:601/1480 train_time:89348ms step_avg:151.18ms
step:602/1480 train_time:89508ms step_avg:151.20ms
step:603/1480 train_time:89669ms step_avg:151.21ms
step:604/1480 train_time:89828ms step_avg:151.23ms
step:605/1480 train_time:89989ms step_avg:151.24ms
step:606/1480 train_time:90151ms step_avg:151.26ms
step:607/1480 train_time:90313ms step_avg:151.28ms
step:608/1480 train_time:90471ms step_avg:151.29ms
step:609/1480 train_time:90632ms step_avg:151.30ms
step:610/1480 train_time:90791ms step_avg:151.32ms
step:611/1480 train_time:90950ms step_avg:151.33ms
step:612/1480 train_time:91111ms step_avg:151.35ms
step:613/1480 train_time:91273ms step_avg:151.36ms
step:614/1480 train_time:91433ms step_avg:151.38ms
step:615/1480 train_time:91592ms step_avg:151.39ms
step:616/1480 train_time:91749ms step_avg:151.40ms
step:617/1480 train_time:91910ms step_avg:151.42ms
step:618/1480 train_time:92068ms step_avg:151.43ms
step:619/1480 train_time:92228ms step_avg:151.44ms
step:620/1480 train_time:92389ms step_avg:151.46ms
step:621/1480 train_time:92549ms step_avg:151.47ms
step:622/1480 train_time:92710ms step_avg:151.49ms
step:623/1480 train_time:92871ms step_avg:151.50ms
step:624/1480 train_time:93030ms step_avg:151.52ms
step:625/1480 train_time:93190ms step_avg:151.53ms
step:625/1480 val_loss:3.6016 train_time:93253ms step_avg:151.63ms
step:626/1480 train_time:93351ms step_avg:151.54ms
step:627/1480 train_time:93509ms step_avg:151.55ms
step:628/1480 train_time:93667ms step_avg:151.56ms
step:629/1480 train_time:93825ms step_avg:151.57ms
step:630/1480 train_time:93983ms step_avg:151.59ms
step:631/1480 train_time:94142ms step_avg:151.60ms
step:632/1480 train_time:94302ms step_avg:151.61ms
step:633/1480 train_time:94462ms step_avg:151.62ms
step:634/1480 train_time:94622ms step_avg:151.64ms
step:635/1480 train_time:94783ms step_avg:151.65ms
step:636/1480 train_time:94943ms step_avg:151.67ms
step:637/1480 train_time:95103ms step_avg:151.68ms
step:638/1480 train_time:95262ms step_avg:151.69ms
step:639/1480 train_time:95421ms step_avg:151.70ms
step:640/1480 train_time:95582ms step_avg:151.72ms
step:641/1480 train_time:95742ms step_avg:151.73ms
step:642/1480 train_time:95902ms step_avg:151.74ms
step:643/1480 train_time:96062ms step_avg:151.76ms
step:644/1480 train_time:96220ms step_avg:151.77ms
step:645/1480 train_time:96380ms step_avg:151.78ms
step:646/1480 train_time:96539ms step_avg:151.79ms
step:647/1480 train_time:96699ms step_avg:151.80ms
step:648/1480 train_time:96862ms step_avg:151.82ms
step:649/1480 train_time:97023ms step_avg:151.84ms
step:650/1480 train_time:97184ms step_avg:151.85ms
step:651/1480 train_time:97343ms step_avg:151.86ms
step:652/1480 train_time:97502ms step_avg:151.87ms
step:653/1480 train_time:97661ms step_avg:151.88ms
step:654/1480 train_time:97821ms step_avg:151.90ms
step:655/1480 train_time:97983ms step_avg:151.91ms
step:656/1480 train_time:98143ms step_avg:151.92ms
step:657/1480 train_time:98304ms step_avg:151.94ms
step:658/1480 train_time:98463ms step_avg:151.95ms
step:659/1480 train_time:98624ms step_avg:151.96ms
step:660/1480 train_time:98786ms step_avg:151.98ms
step:661/1480 train_time:98949ms step_avg:151.99ms
step:662/1480 train_time:99109ms step_avg:152.01ms
step:663/1480 train_time:99269ms step_avg:152.02ms
step:664/1480 train_time:99431ms step_avg:152.04ms
step:665/1480 train_time:99591ms step_avg:152.05ms
step:666/1480 train_time:99751ms step_avg:152.06ms
step:667/1480 train_time:99911ms step_avg:152.07ms
step:668/1480 train_time:100073ms step_avg:152.09ms
step:669/1480 train_time:100237ms step_avg:152.10ms
step:670/1480 train_time:100400ms step_avg:152.12ms
step:671/1480 train_time:100562ms step_avg:152.14ms
step:672/1480 train_time:100723ms step_avg:152.15ms
step:673/1480 train_time:100885ms step_avg:152.16ms
step:674/1480 train_time:101046ms step_avg:152.18ms
step:675/1480 train_time:101208ms step_avg:152.19ms
step:676/1480 train_time:101370ms step_avg:152.21ms
step:677/1480 train_time:101529ms step_avg:152.22ms
step:678/1480 train_time:101690ms step_avg:152.23ms
step:679/1480 train_time:101852ms step_avg:152.24ms
step:680/1480 train_time:102013ms step_avg:152.26ms
step:681/1480 train_time:102173ms step_avg:152.27ms
step:682/1480 train_time:102337ms step_avg:152.29ms
step:683/1480 train_time:102500ms step_avg:152.30ms
step:684/1480 train_time:102662ms step_avg:152.32ms
step:685/1480 train_time:102825ms step_avg:152.33ms
step:686/1480 train_time:102986ms step_avg:152.35ms
step:687/1480 train_time:103146ms step_avg:152.36ms
step:688/1480 train_time:103308ms step_avg:152.37ms
step:689/1480 train_time:103471ms step_avg:152.39ms
step:690/1480 train_time:103637ms step_avg:152.41ms
step:691/1480 train_time:103799ms step_avg:152.42ms
step:692/1480 train_time:103961ms step_avg:152.44ms
step:693/1480 train_time:104123ms step_avg:152.45ms
step:694/1480 train_time:104285ms step_avg:152.46ms
step:695/1480 train_time:104446ms step_avg:152.48ms
step:696/1480 train_time:104607ms step_avg:152.49ms
step:697/1480 train_time:104769ms step_avg:152.50ms
step:698/1480 train_time:104928ms step_avg:152.51ms
step:699/1480 train_time:105091ms step_avg:152.53ms
step:700/1480 train_time:105253ms step_avg:152.54ms
step:701/1480 train_time:105413ms step_avg:152.55ms
step:702/1480 train_time:105575ms step_avg:152.56ms
step:703/1480 train_time:105736ms step_avg:152.58ms
step:704/1480 train_time:105897ms step_avg:152.59ms
step:705/1480 train_time:106061ms step_avg:152.61ms
step:706/1480 train_time:106223ms step_avg:152.62ms
step:707/1480 train_time:106384ms step_avg:152.63ms
step:708/1480 train_time:106545ms step_avg:152.64ms
step:709/1480 train_time:106707ms step_avg:152.66ms
step:710/1480 train_time:106868ms step_avg:152.67ms
step:711/1480 train_time:107030ms step_avg:152.68ms
step:712/1480 train_time:107196ms step_avg:152.70ms
step:713/1480 train_time:107361ms step_avg:152.72ms
step:714/1480 train_time:107523ms step_avg:152.73ms
step:715/1480 train_time:107684ms step_avg:152.74ms
step:716/1480 train_time:107844ms step_avg:152.75ms
step:717/1480 train_time:108007ms step_avg:152.77ms
step:718/1480 train_time:108166ms step_avg:152.78ms
step:719/1480 train_time:108324ms step_avg:152.78ms
step:720/1480 train_time:108487ms step_avg:152.80ms
step:721/1480 train_time:108648ms step_avg:152.81ms
step:722/1480 train_time:108809ms step_avg:152.82ms
step:723/1480 train_time:108969ms step_avg:152.83ms
step:724/1480 train_time:109130ms step_avg:152.84ms
step:725/1480 train_time:109294ms step_avg:152.86ms
step:726/1480 train_time:109457ms step_avg:152.87ms
step:727/1480 train_time:109620ms step_avg:152.89ms
step:728/1480 train_time:109782ms step_avg:152.90ms
step:729/1480 train_time:109943ms step_avg:152.91ms
step:730/1480 train_time:110106ms step_avg:152.92ms
step:731/1480 train_time:110266ms step_avg:152.93ms
step:732/1480 train_time:110425ms step_avg:152.94ms
step:733/1480 train_time:110586ms step_avg:152.95ms
step:734/1480 train_time:110748ms step_avg:152.97ms
step:735/1480 train_time:110909ms step_avg:152.98ms
step:736/1480 train_time:111071ms step_avg:152.99ms
step:737/1480 train_time:111231ms step_avg:153.00ms
step:738/1480 train_time:111392ms step_avg:153.01ms
step:739/1480 train_time:111554ms step_avg:153.02ms
step:740/1480 train_time:111720ms step_avg:153.04ms
step:741/1480 train_time:111883ms step_avg:153.06ms
step:742/1480 train_time:112046ms step_avg:153.07ms
step:743/1480 train_time:112207ms step_avg:153.08ms
step:744/1480 train_time:112371ms step_avg:153.09ms
step:745/1480 train_time:112536ms step_avg:153.11ms
step:746/1480 train_time:112695ms step_avg:153.12ms
step:747/1480 train_time:112858ms step_avg:153.13ms
step:748/1480 train_time:113024ms step_avg:153.15ms
step:749/1480 train_time:113188ms step_avg:153.16ms
step:750/1480 train_time:113347ms step_avg:153.17ms
step:750/1480 val_loss:3.5457 train_time:113412ms step_avg:153.26ms
step:751/1480 train_time:113512ms step_avg:153.19ms
step:752/1480 train_time:113673ms step_avg:153.20ms
step:753/1480 train_time:113833ms step_avg:153.21ms
step:754/1480 train_time:113992ms step_avg:153.22ms
step:755/1480 train_time:114153ms step_avg:153.23ms
step:756/1480 train_time:114315ms step_avg:153.24ms
step:757/1480 train_time:114479ms step_avg:153.25ms
step:758/1480 train_time:114640ms step_avg:153.26ms
step:759/1480 train_time:114803ms step_avg:153.28ms
step:760/1480 train_time:114965ms step_avg:153.29ms
step:761/1480 train_time:115128ms step_avg:153.30ms
step:762/1480 train_time:115289ms step_avg:153.31ms
step:763/1480 train_time:115450ms step_avg:153.32ms
step:764/1480 train_time:115612ms step_avg:153.33ms
step:765/1480 train_time:115773ms step_avg:153.34ms
step:766/1480 train_time:115935ms step_avg:153.35ms
step:767/1480 train_time:116096ms step_avg:153.36ms
step:768/1480 train_time:116258ms step_avg:153.37ms
step:769/1480 train_time:116424ms step_avg:153.39ms
step:770/1480 train_time:116586ms step_avg:153.40ms
step:771/1480 train_time:116752ms step_avg:153.42ms
step:772/1480 train_time:116914ms step_avg:153.43ms
step:773/1480 train_time:117075ms step_avg:153.44ms
step:774/1480 train_time:117236ms step_avg:153.45ms
step:775/1480 train_time:117401ms step_avg:153.47ms
step:776/1480 train_time:117565ms step_avg:153.48ms
step:777/1480 train_time:117731ms step_avg:153.49ms
step:778/1480 train_time:117892ms step_avg:153.51ms
step:779/1480 train_time:118054ms step_avg:153.52ms
step:780/1480 train_time:118219ms step_avg:153.53ms
step:781/1480 train_time:118382ms step_avg:153.54ms
step:782/1480 train_time:118547ms step_avg:153.56ms
step:783/1480 train_time:118708ms step_avg:153.57ms
step:784/1480 train_time:118871ms step_avg:153.58ms
step:785/1480 train_time:119033ms step_avg:153.59ms
step:786/1480 train_time:119199ms step_avg:153.61ms
step:787/1480 train_time:119362ms step_avg:153.62ms
step:788/1480 train_time:119527ms step_avg:153.63ms
step:789/1480 train_time:119689ms step_avg:153.64ms
step:790/1480 train_time:119855ms step_avg:153.66ms
step:791/1480 train_time:120025ms step_avg:153.68ms
step:792/1480 train_time:120189ms step_avg:153.69ms
step:793/1480 train_time:120350ms step_avg:153.70ms
step:794/1480 train_time:120515ms step_avg:153.72ms
step:795/1480 train_time:120681ms step_avg:153.73ms
step:796/1480 train_time:120849ms step_avg:153.75ms
step:797/1480 train_time:121013ms step_avg:153.77ms
step:798/1480 train_time:121176ms step_avg:153.78ms
step:799/1480 train_time:121344ms step_avg:153.80ms
step:800/1480 train_time:121508ms step_avg:153.81ms
step:801/1480 train_time:121671ms step_avg:153.82ms
step:802/1480 train_time:121838ms step_avg:153.84ms
step:803/1480 train_time:122001ms step_avg:153.85ms
step:804/1480 train_time:122163ms step_avg:153.86ms
step:805/1480 train_time:122329ms step_avg:153.87ms
step:806/1480 train_time:122491ms step_avg:153.88ms
step:807/1480 train_time:122651ms step_avg:153.89ms
step:808/1480 train_time:122814ms step_avg:153.90ms
step:809/1480 train_time:122976ms step_avg:153.91ms
step:810/1480 train_time:123139ms step_avg:153.92ms
step:811/1480 train_time:123303ms step_avg:153.94ms
step:812/1480 train_time:123467ms step_avg:153.95ms
step:813/1480 train_time:123628ms step_avg:153.96ms
step:814/1480 train_time:123791ms step_avg:153.97ms
step:815/1480 train_time:123952ms step_avg:153.98ms
step:816/1480 train_time:124119ms step_avg:153.99ms
step:817/1480 train_time:124282ms step_avg:154.00ms
step:818/1480 train_time:124445ms step_avg:154.02ms
step:819/1480 train_time:124608ms step_avg:154.03ms
step:820/1480 train_time:124773ms step_avg:154.04ms
step:821/1480 train_time:124934ms step_avg:154.05ms
step:822/1480 train_time:125100ms step_avg:154.06ms
step:823/1480 train_time:125262ms step_avg:154.07ms
step:824/1480 train_time:125425ms step_avg:154.08ms
step:825/1480 train_time:125589ms step_avg:154.10ms
step:826/1480 train_time:125755ms step_avg:154.11ms
step:827/1480 train_time:125920ms step_avg:154.12ms
step:828/1480 train_time:126084ms step_avg:154.14ms
step:829/1480 train_time:126249ms step_avg:154.15ms
step:830/1480 train_time:126413ms step_avg:154.16ms
step:831/1480 train_time:126576ms step_avg:154.17ms
step:832/1480 train_time:126740ms step_avg:154.18ms
step:833/1480 train_time:126905ms step_avg:154.20ms
step:834/1480 train_time:127069ms step_avg:154.21ms
step:835/1480 train_time:127231ms step_avg:154.22ms
step:836/1480 train_time:127396ms step_avg:154.23ms
step:837/1480 train_time:127558ms step_avg:154.24ms
step:838/1480 train_time:127724ms step_avg:154.26ms
step:839/1480 train_time:127887ms step_avg:154.27ms
step:840/1480 train_time:128049ms step_avg:154.28ms
step:841/1480 train_time:128210ms step_avg:154.28ms
step:842/1480 train_time:128374ms step_avg:154.30ms
step:843/1480 train_time:128534ms step_avg:154.30ms
step:844/1480 train_time:128696ms step_avg:154.31ms
step:845/1480 train_time:128860ms step_avg:154.32ms
step:846/1480 train_time:129027ms step_avg:154.34ms
step:847/1480 train_time:129190ms step_avg:154.35ms
step:848/1480 train_time:129352ms step_avg:154.36ms
step:849/1480 train_time:129515ms step_avg:154.37ms
step:850/1480 train_time:129676ms step_avg:154.38ms
step:851/1480 train_time:129842ms step_avg:154.39ms
step:852/1480 train_time:130005ms step_avg:154.40ms
step:853/1480 train_time:130168ms step_avg:154.41ms
step:854/1480 train_time:130332ms step_avg:154.42ms
step:855/1480 train_time:130494ms step_avg:154.43ms
step:856/1480 train_time:130654ms step_avg:154.44ms
step:857/1480 train_time:130821ms step_avg:154.45ms
step:858/1480 train_time:130986ms step_avg:154.46ms
step:859/1480 train_time:131150ms step_avg:154.48ms
step:860/1480 train_time:131311ms step_avg:154.48ms
step:861/1480 train_time:131477ms step_avg:154.50ms
step:862/1480 train_time:131647ms step_avg:154.52ms
step:863/1480 train_time:131815ms step_avg:154.53ms
step:864/1480 train_time:131978ms step_avg:154.54ms
step:865/1480 train_time:132140ms step_avg:154.55ms
step:866/1480 train_time:132307ms step_avg:154.56ms
step:867/1480 train_time:132470ms step_avg:154.57ms
step:868/1480 train_time:132630ms step_avg:154.58ms
step:869/1480 train_time:132791ms step_avg:154.59ms
step:870/1480 train_time:132955ms step_avg:154.60ms
step:871/1480 train_time:133119ms step_avg:154.61ms
step:872/1480 train_time:133285ms step_avg:154.62ms
step:873/1480 train_time:133449ms step_avg:154.63ms
step:874/1480 train_time:133614ms step_avg:154.65ms
step:875/1480 train_time:133780ms step_avg:154.66ms
step:875/1480 val_loss:3.5022 train_time:133844ms step_avg:154.73ms
step:876/1480 train_time:133944ms step_avg:154.67ms
step:877/1480 train_time:134108ms step_avg:154.68ms
step:878/1480 train_time:134271ms step_avg:154.69ms
step:879/1480 train_time:134435ms step_avg:154.70ms
step:880/1480 train_time:134599ms step_avg:154.71ms
step:881/1480 train_time:134762ms step_avg:154.72ms
step:882/1480 train_time:134927ms step_avg:154.73ms
step:883/1480 train_time:135093ms step_avg:154.75ms
step:884/1480 train_time:135262ms step_avg:154.76ms
step:885/1480 train_time:135426ms step_avg:154.77ms
step:886/1480 train_time:135592ms step_avg:154.79ms
step:887/1480 train_time:135760ms step_avg:154.80ms
step:888/1480 train_time:135932ms step_avg:154.82ms
step:889/1480 train_time:136101ms step_avg:154.84ms
step:890/1480 train_time:136263ms step_avg:154.84ms
step:891/1480 train_time:136428ms step_avg:154.86ms
step:892/1480 train_time:136593ms step_avg:154.87ms
step:893/1480 train_time:136756ms step_avg:154.88ms
step:894/1480 train_time:136924ms step_avg:154.89ms
step:895/1480 train_time:137089ms step_avg:154.90ms
step:896/1480 train_time:137253ms step_avg:154.91ms
step:897/1480 train_time:137420ms step_avg:154.93ms
step:898/1480 train_time:137588ms step_avg:154.94ms
step:899/1480 train_time:137752ms step_avg:154.95ms
step:900/1480 train_time:137915ms step_avg:154.96ms
step:901/1480 train_time:138082ms step_avg:154.97ms
step:902/1480 train_time:138245ms step_avg:154.98ms
step:903/1480 train_time:138416ms step_avg:155.00ms
step:904/1480 train_time:138584ms step_avg:155.02ms
step:905/1480 train_time:138746ms step_avg:155.02ms
step:906/1480 train_time:138911ms step_avg:155.03ms
step:907/1480 train_time:139079ms step_avg:155.05ms
step:908/1480 train_time:139242ms step_avg:155.06ms
step:909/1480 train_time:139407ms step_avg:155.07ms
step:910/1480 train_time:139578ms step_avg:155.09ms
step:911/1480 train_time:139744ms step_avg:155.10ms
step:912/1480 train_time:139909ms step_avg:155.11ms
step:913/1480 train_time:140077ms step_avg:155.12ms
step:914/1480 train_time:140244ms step_avg:155.14ms
step:915/1480 train_time:140412ms step_avg:155.15ms
step:916/1480 train_time:140579ms step_avg:155.16ms
step:917/1480 train_time:140742ms step_avg:155.17ms
step:918/1480 train_time:140910ms step_avg:155.19ms
step:919/1480 train_time:141081ms step_avg:155.21ms
step:920/1480 train_time:141247ms step_avg:155.22ms
step:921/1480 train_time:141413ms step_avg:155.23ms
step:922/1480 train_time:141582ms step_avg:155.24ms
step:923/1480 train_time:141744ms step_avg:155.25ms
step:924/1480 train_time:141908ms step_avg:155.26ms
step:925/1480 train_time:142075ms step_avg:155.27ms
step:926/1480 train_time:142239ms step_avg:155.28ms
step:927/1480 train_time:142404ms step_avg:155.29ms
step:928/1480 train_time:142571ms step_avg:155.31ms
step:929/1480 train_time:142737ms step_avg:155.32ms
step:930/1480 train_time:142904ms step_avg:155.33ms
step:931/1480 train_time:143067ms step_avg:155.34ms
step:932/1480 train_time:143232ms step_avg:155.35ms
step:933/1480 train_time:143400ms step_avg:155.36ms
step:934/1480 train_time:143566ms step_avg:155.37ms
step:935/1480 train_time:143736ms step_avg:155.39ms
step:936/1480 train_time:143904ms step_avg:155.40ms
step:937/1480 train_time:144075ms step_avg:155.42ms
step:938/1480 train_time:144237ms step_avg:155.43ms
step:939/1480 train_time:144406ms step_avg:155.44ms
step:940/1480 train_time:144574ms step_avg:155.46ms
step:941/1480 train_time:144738ms step_avg:155.47ms
step:942/1480 train_time:144904ms step_avg:155.48ms
step:943/1480 train_time:145074ms step_avg:155.49ms
step:944/1480 train_time:145245ms step_avg:155.51ms
step:945/1480 train_time:145408ms step_avg:155.52ms
step:946/1480 train_time:145578ms step_avg:155.53ms
step:947/1480 train_time:145745ms step_avg:155.54ms
step:948/1480 train_time:145910ms step_avg:155.55ms
step:949/1480 train_time:146076ms step_avg:155.57ms
step:950/1480 train_time:146241ms step_avg:155.58ms
step:951/1480 train_time:146407ms step_avg:155.59ms
step:952/1480 train_time:146572ms step_avg:155.60ms
step:953/1480 train_time:146740ms step_avg:155.61ms
step:954/1480 train_time:146906ms step_avg:155.62ms
step:955/1480 train_time:147068ms step_avg:155.63ms
step:956/1480 train_time:147232ms step_avg:155.64ms
step:957/1480 train_time:147402ms step_avg:155.65ms
step:958/1480 train_time:147570ms step_avg:155.67ms
step:959/1480 train_time:147734ms step_avg:155.67ms
step:960/1480 train_time:147903ms step_avg:155.69ms
step:961/1480 train_time:148068ms step_avg:155.70ms
step:962/1480 train_time:148231ms step_avg:155.70ms
step:963/1480 train_time:148398ms step_avg:155.72ms
step:964/1480 train_time:148567ms step_avg:155.73ms
step:965/1480 train_time:148730ms step_avg:155.74ms
step:966/1480 train_time:148895ms step_avg:155.75ms
step:967/1480 train_time:149059ms step_avg:155.76ms
step:968/1480 train_time:149223ms step_avg:155.77ms
step:969/1480 train_time:149390ms step_avg:155.78ms
step:970/1480 train_time:149552ms step_avg:155.78ms
step:971/1480 train_time:149716ms step_avg:155.79ms
step:972/1480 train_time:149883ms step_avg:155.80ms
step:973/1480 train_time:150047ms step_avg:155.81ms
step:974/1480 train_time:150215ms step_avg:155.82ms
step:975/1480 train_time:150381ms step_avg:155.84ms
step:976/1480 train_time:150547ms step_avg:155.85ms
step:977/1480 train_time:150709ms step_avg:155.85ms
step:978/1480 train_time:150876ms step_avg:155.86ms
step:979/1480 train_time:151043ms step_avg:155.88ms
step:980/1480 train_time:151208ms step_avg:155.88ms
step:981/1480 train_time:151378ms step_avg:155.90ms
step:982/1480 train_time:151540ms step_avg:155.91ms
step:983/1480 train_time:151705ms step_avg:155.91ms
step:984/1480 train_time:151869ms step_avg:155.92ms
step:985/1480 train_time:152037ms step_avg:155.94ms
step:986/1480 train_time:152203ms step_avg:155.95ms
step:987/1480 train_time:152368ms step_avg:155.95ms
step:988/1480 train_time:152535ms step_avg:155.97ms
step:989/1480 train_time:152701ms step_avg:155.98ms
step:990/1480 train_time:152870ms step_avg:155.99ms
step:991/1480 train_time:153038ms step_avg:156.00ms
step:992/1480 train_time:153212ms step_avg:156.02ms
step:993/1480 train_time:153390ms step_avg:156.04ms
step:994/1480 train_time:153555ms step_avg:156.05ms
step:995/1480 train_time:153721ms step_avg:156.06ms
step:996/1480 train_time:153885ms step_avg:156.07ms
step:997/1480 train_time:154049ms step_avg:156.08ms
step:998/1480 train_time:154211ms step_avg:156.08ms
step:999/1480 train_time:154380ms step_avg:156.10ms
step:1000/1480 train_time:154547ms step_avg:156.11ms
step:1000/1480 val_loss:3.4393 train_time:154615ms step_avg:156.18ms
step:1001/1480 train_time:154719ms step_avg:156.12ms
step:1002/1480 train_time:154885ms step_avg:156.13ms
step:1003/1480 train_time:155058ms step_avg:156.15ms
step:1004/1480 train_time:155227ms step_avg:156.16ms
step:1005/1480 train_time:155395ms step_avg:156.18ms
step:1006/1480 train_time:155563ms step_avg:156.19ms
step:1007/1480 train_time:155728ms step_avg:156.20ms
step:1008/1480 train_time:155896ms step_avg:156.21ms
step:1009/1480 train_time:156069ms step_avg:156.22ms
step:1010/1480 train_time:156234ms step_avg:156.23ms
step:1011/1480 train_time:156400ms step_avg:156.24ms
step:1012/1480 train_time:156565ms step_avg:156.25ms
step:1013/1480 train_time:156735ms step_avg:156.27ms
step:1014/1480 train_time:156902ms step_avg:156.28ms
step:1015/1480 train_time:157072ms step_avg:156.29ms
step:1016/1480 train_time:157241ms step_avg:156.30ms
step:1017/1480 train_time:157412ms step_avg:156.32ms
step:1018/1480 train_time:157581ms step_avg:156.33ms
step:1019/1480 train_time:157748ms step_avg:156.34ms
step:1020/1480 train_time:157917ms step_avg:156.35ms
step:1021/1480 train_time:158082ms step_avg:156.36ms
step:1022/1480 train_time:158249ms step_avg:156.37ms
step:1023/1480 train_time:158418ms step_avg:156.38ms
step:1024/1480 train_time:158584ms step_avg:156.39ms
step:1025/1480 train_time:158756ms step_avg:156.41ms
step:1026/1480 train_time:158922ms step_avg:156.42ms
step:1027/1480 train_time:159089ms step_avg:156.43ms
step:1028/1480 train_time:159262ms step_avg:156.45ms
step:1029/1480 train_time:159438ms step_avg:156.47ms
step:1030/1480 train_time:159605ms step_avg:156.48ms
step:1031/1480 train_time:159769ms step_avg:156.48ms
step:1032/1480 train_time:159943ms step_avg:156.50ms
step:1033/1480 train_time:160108ms step_avg:156.51ms
step:1034/1480 train_time:160275ms step_avg:156.52ms
step:1035/1480 train_time:160443ms step_avg:156.53ms
step:1036/1480 train_time:160608ms step_avg:156.54ms
step:1037/1480 train_time:160775ms step_avg:156.55ms
step:1038/1480 train_time:160944ms step_avg:156.56ms
step:1039/1480 train_time:161115ms step_avg:156.57ms
step:1040/1480 train_time:161281ms step_avg:156.58ms
step:1041/1480 train_time:161447ms step_avg:156.59ms
step:1042/1480 train_time:161611ms step_avg:156.60ms
step:1043/1480 train_time:161775ms step_avg:156.61ms
step:1044/1480 train_time:161943ms step_avg:156.62ms
step:1045/1480 train_time:162113ms step_avg:156.63ms
step:1046/1480 train_time:162281ms step_avg:156.64ms
step:1047/1480 train_time:162448ms step_avg:156.65ms
step:1048/1480 train_time:162616ms step_avg:156.66ms
step:1049/1480 train_time:162782ms step_avg:156.67ms
step:1050/1480 train_time:162950ms step_avg:156.68ms
step:1051/1480 train_time:163121ms step_avg:156.70ms
step:1052/1480 train_time:163289ms step_avg:156.71ms
step:1053/1480 train_time:163457ms step_avg:156.72ms
step:1054/1480 train_time:163624ms step_avg:156.73ms
step:1055/1480 train_time:163789ms step_avg:156.74ms
step:1056/1480 train_time:163954ms step_avg:156.74ms
step:1057/1480 train_time:164122ms step_avg:156.75ms
step:1058/1480 train_time:164291ms step_avg:156.77ms
step:1059/1480 train_time:164463ms step_avg:156.78ms
step:1060/1480 train_time:164633ms step_avg:156.79ms
step:1061/1480 train_time:164798ms step_avg:156.80ms
step:1062/1480 train_time:164965ms step_avg:156.81ms
step:1063/1480 train_time:165129ms step_avg:156.82ms
step:1064/1480 train_time:165293ms step_avg:156.82ms
step:1065/1480 train_time:165461ms step_avg:156.83ms
step:1066/1480 train_time:165629ms step_avg:156.85ms
step:1067/1480 train_time:165800ms step_avg:156.86ms
step:1068/1480 train_time:165967ms step_avg:156.87ms
step:1069/1480 train_time:166138ms step_avg:156.88ms
step:1070/1480 train_time:166304ms step_avg:156.89ms
step:1071/1480 train_time:166476ms step_avg:156.90ms
step:1072/1480 train_time:166642ms step_avg:156.91ms
step:1073/1480 train_time:166805ms step_avg:156.92ms
step:1074/1480 train_time:166972ms step_avg:156.93ms
step:1075/1480 train_time:167144ms step_avg:156.94ms
step:1076/1480 train_time:167312ms step_avg:156.95ms
step:1077/1480 train_time:167477ms step_avg:156.96ms
step:1078/1480 train_time:167651ms step_avg:156.98ms
step:1079/1480 train_time:167823ms step_avg:156.99ms
step:1080/1480 train_time:167993ms step_avg:157.00ms
step:1081/1480 train_time:168160ms step_avg:157.01ms
step:1082/1480 train_time:168327ms step_avg:157.02ms
step:1083/1480 train_time:168492ms step_avg:157.03ms
step:1084/1480 train_time:168659ms step_avg:157.04ms
step:1085/1480 train_time:168827ms step_avg:157.05ms
step:1086/1480 train_time:168993ms step_avg:157.06ms
step:1087/1480 train_time:169160ms step_avg:157.07ms
step:1088/1480 train_time:169330ms step_avg:157.08ms
step:1089/1480 train_time:169504ms step_avg:157.09ms
step:1090/1480 train_time:169677ms step_avg:157.11ms
step:1091/1480 train_time:169845ms step_avg:157.12ms
step:1092/1480 train_time:170013ms step_avg:157.13ms
step:1093/1480 train_time:170181ms step_avg:157.14ms
step:1094/1480 train_time:170347ms step_avg:157.15ms
step:1095/1480 train_time:170512ms step_avg:157.15ms
step:1096/1480 train_time:170680ms step_avg:157.16ms
step:1097/1480 train_time:170847ms step_avg:157.17ms
step:1098/1480 train_time:171020ms step_avg:157.19ms
step:1099/1480 train_time:171191ms step_avg:157.20ms
step:1100/1480 train_time:171363ms step_avg:157.21ms
step:1101/1480 train_time:171534ms step_avg:157.23ms
step:1102/1480 train_time:171705ms step_avg:157.24ms
step:1103/1480 train_time:171881ms step_avg:157.26ms
step:1104/1480 train_time:172048ms step_avg:157.27ms
step:1105/1480 train_time:172221ms step_avg:157.28ms
step:1106/1480 train_time:172389ms step_avg:157.29ms
step:1107/1480 train_time:172558ms step_avg:157.30ms
step:1108/1480 train_time:172725ms step_avg:157.31ms
step:1109/1480 train_time:172890ms step_avg:157.32ms
step:1110/1480 train_time:173055ms step_avg:157.32ms
step:1111/1480 train_time:173222ms step_avg:157.33ms
step:1112/1480 train_time:173392ms step_avg:157.34ms
step:1113/1480 train_time:173572ms step_avg:157.36ms
step:1114/1480 train_time:173745ms step_avg:157.38ms
step:1115/1480 train_time:173917ms step_avg:157.39ms
step:1116/1480 train_time:174082ms step_avg:157.40ms
step:1117/1480 train_time:174254ms step_avg:157.41ms
step:1118/1480 train_time:174430ms step_avg:157.43ms
step:1119/1480 train_time:174597ms step_avg:157.44ms
step:1120/1480 train_time:174766ms step_avg:157.45ms
step:1121/1480 train_time:174935ms step_avg:157.46ms
step:1122/1480 train_time:175101ms step_avg:157.47ms
step:1123/1480 train_time:175267ms step_avg:157.47ms
step:1124/1480 train_time:175437ms step_avg:157.48ms
step:1125/1480 train_time:175605ms step_avg:157.49ms
step:1125/1480 val_loss:3.3836 train_time:175673ms step_avg:157.55ms
step:1126/1480 train_time:175774ms step_avg:157.50ms
step:1127/1480 train_time:175945ms step_avg:157.52ms
step:1128/1480 train_time:176115ms step_avg:157.53ms
step:1129/1480 train_time:176290ms step_avg:157.54ms
step:1130/1480 train_time:176459ms step_avg:157.55ms
step:1131/1480 train_time:176637ms step_avg:157.57ms
step:1132/1480 train_time:176803ms step_avg:157.58ms
step:1133/1480 train_time:176976ms step_avg:157.59ms
step:1134/1480 train_time:177147ms step_avg:157.60ms
step:1135/1480 train_time:177315ms step_avg:157.61ms
step:1136/1480 train_time:177486ms step_avg:157.63ms
step:1137/1480 train_time:177655ms step_avg:157.64ms
step:1138/1480 train_time:177827ms step_avg:157.65ms
step:1139/1480 train_time:177996ms step_avg:157.66ms
step:1140/1480 train_time:178165ms step_avg:157.67ms
step:1141/1480 train_time:178336ms step_avg:157.68ms
step:1142/1480 train_time:178505ms step_avg:157.69ms
step:1143/1480 train_time:178675ms step_avg:157.70ms
step:1144/1480 train_time:178843ms step_avg:157.71ms
step:1145/1480 train_time:179008ms step_avg:157.72ms
step:1146/1480 train_time:179177ms step_avg:157.73ms
step:1147/1480 train_time:179347ms step_avg:157.74ms
step:1148/1480 train_time:179515ms step_avg:157.75ms
step:1149/1480 train_time:179687ms step_avg:157.76ms
step:1150/1480 train_time:179856ms step_avg:157.77ms
step:1151/1480 train_time:180029ms step_avg:157.78ms
step:1152/1480 train_time:180200ms step_avg:157.79ms
step:1153/1480 train_time:180374ms step_avg:157.81ms
step:1154/1480 train_time:180540ms step_avg:157.81ms
step:1155/1480 train_time:180713ms step_avg:157.83ms
step:1156/1480 train_time:180892ms step_avg:157.85ms
step:1157/1480 train_time:181061ms step_avg:157.86ms
step:1158/1480 train_time:181229ms step_avg:157.86ms
step:1159/1480 train_time:181395ms step_avg:157.87ms
step:1160/1480 train_time:181561ms step_avg:157.88ms
step:1161/1480 train_time:181731ms step_avg:157.89ms
step:1162/1480 train_time:181900ms step_avg:157.90ms
step:1163/1480 train_time:182070ms step_avg:157.91ms
step:1164/1480 train_time:182239ms step_avg:157.92ms
step:1165/1480 train_time:182405ms step_avg:157.93ms
step:1166/1480 train_time:182574ms step_avg:157.94ms
step:1167/1480 train_time:182742ms step_avg:157.95ms
step:1168/1480 train_time:182910ms step_avg:157.95ms
step:1169/1480 train_time:183078ms step_avg:157.96ms
step:1170/1480 train_time:183247ms step_avg:157.97ms
step:1171/1480 train_time:183415ms step_avg:157.98ms
step:1172/1480 train_time:183582ms step_avg:157.99ms
step:1173/1480 train_time:183754ms step_avg:158.00ms
step:1174/1480 train_time:183935ms step_avg:158.02ms
step:1175/1480 train_time:184106ms step_avg:158.03ms
step:1176/1480 train_time:184277ms step_avg:158.04ms
step:1177/1480 train_time:184456ms step_avg:158.06ms
step:1178/1480 train_time:184624ms step_avg:158.07ms
step:1179/1480 train_time:184790ms step_avg:158.08ms
step:1180/1480 train_time:184970ms step_avg:158.09ms
step:1181/1480 train_time:185139ms step_avg:158.10ms
step:1182/1480 train_time:185307ms step_avg:158.11ms
step:1183/1480 train_time:185477ms step_avg:158.12ms
step:1184/1480 train_time:185647ms step_avg:158.13ms
step:1185/1480 train_time:185817ms step_avg:158.14ms
step:1186/1480 train_time:185988ms step_avg:158.15ms
step:1187/1480 train_time:186173ms step_avg:158.18ms
step:1188/1480 train_time:186339ms step_avg:158.18ms
step:1189/1480 train_time:186510ms step_avg:158.19ms
step:1190/1480 train_time:186677ms step_avg:158.20ms
step:1191/1480 train_time:186850ms step_avg:158.21ms
step:1192/1480 train_time:187015ms step_avg:158.22ms
step:1193/1480 train_time:187182ms step_avg:158.23ms
step:1194/1480 train_time:187351ms step_avg:158.24ms
step:1195/1480 train_time:187524ms step_avg:158.25ms
step:1196/1480 train_time:187707ms step_avg:158.27ms
step:1197/1480 train_time:187877ms step_avg:158.28ms
step:1198/1480 train_time:188058ms step_avg:158.30ms
step:1199/1480 train_time:188229ms step_avg:158.31ms
step:1200/1480 train_time:188397ms step_avg:158.32ms
step:1201/1480 train_time:188565ms step_avg:158.33ms
step:1202/1480 train_time:188747ms step_avg:158.34ms
step:1203/1480 train_time:188923ms step_avg:158.36ms
step:1204/1480 train_time:189098ms step_avg:158.37ms
step:1205/1480 train_time:189267ms step_avg:158.38ms
step:1206/1480 train_time:189434ms step_avg:158.39ms
step:1207/1480 train_time:189605ms step_avg:158.40ms
step:1208/1480 train_time:189774ms step_avg:158.41ms
step:1209/1480 train_time:189946ms step_avg:158.42ms
step:1210/1480 train_time:190119ms step_avg:158.43ms
step:1211/1480 train_time:190293ms step_avg:158.45ms
step:1212/1480 train_time:190465ms step_avg:158.46ms
step:1213/1480 train_time:190637ms step_avg:158.47ms
step:1214/1480 train_time:190814ms step_avg:158.48ms
step:1215/1480 train_time:190987ms step_avg:158.50ms
step:1216/1480 train_time:191157ms step_avg:158.50ms
step:1217/1480 train_time:191330ms step_avg:158.52ms
step:1218/1480 train_time:191500ms step_avg:158.53ms
step:1219/1480 train_time:191678ms step_avg:158.54ms
step:1220/1480 train_time:191848ms step_avg:158.55ms
step:1221/1480 train_time:192016ms step_avg:158.56ms
step:1222/1480 train_time:192184ms step_avg:158.57ms
step:1223/1480 train_time:192355ms step_avg:158.58ms
step:1224/1480 train_time:192534ms step_avg:158.60ms
step:1225/1480 train_time:192707ms step_avg:158.61ms
step:1226/1480 train_time:192878ms step_avg:158.62ms
step:1227/1480 train_time:193051ms step_avg:158.63ms
step:1228/1480 train_time:193221ms step_avg:158.64ms
step:1229/1480 train_time:193394ms step_avg:158.65ms
step:1230/1480 train_time:193575ms step_avg:158.67ms
step:1231/1480 train_time:193750ms step_avg:158.68ms
step:1232/1480 train_time:193924ms step_avg:158.69ms
step:1233/1480 train_time:194094ms step_avg:158.70ms
step:1234/1480 train_time:194266ms step_avg:158.71ms
step:1235/1480 train_time:194440ms step_avg:158.73ms
step:1236/1480 train_time:194609ms step_avg:158.73ms
step:1237/1480 train_time:194779ms step_avg:158.74ms
step:1238/1480 train_time:194964ms step_avg:158.77ms
step:1239/1480 train_time:195134ms step_avg:158.77ms
step:1240/1480 train_time:195304ms step_avg:158.78ms
step:1241/1480 train_time:195476ms step_avg:158.79ms
step:1242/1480 train_time:195646ms step_avg:158.80ms
step:1243/1480 train_time:195818ms step_avg:158.81ms
step:1244/1480 train_time:195984ms step_avg:158.82ms
step:1245/1480 train_time:196153ms step_avg:158.83ms
step:1246/1480 train_time:196322ms step_avg:158.84ms
step:1247/1480 train_time:196491ms step_avg:158.84ms
step:1248/1480 train_time:196660ms step_avg:158.85ms
step:1249/1480 train_time:196829ms step_avg:158.86ms
step:1250/1480 train_time:196998ms step_avg:158.87ms
step:1250/1480 val_loss:3.3342 train_time:197069ms step_avg:158.93ms
step:1251/1480 train_time:197181ms step_avg:158.89ms
step:1252/1480 train_time:197351ms step_avg:158.90ms
step:1253/1480 train_time:197519ms step_avg:158.90ms
step:1254/1480 train_time:197691ms step_avg:158.92ms
step:1255/1480 train_time:197877ms step_avg:158.94ms
step:1256/1480 train_time:198050ms step_avg:158.95ms
step:1257/1480 train_time:198219ms step_avg:158.96ms
step:1258/1480 train_time:198396ms step_avg:158.97ms
step:1259/1480 train_time:198567ms step_avg:158.98ms
step:1260/1480 train_time:198736ms step_avg:158.99ms
step:1261/1480 train_time:198909ms step_avg:159.00ms
step:1262/1480 train_time:199085ms step_avg:159.01ms
step:1263/1480 train_time:199260ms step_avg:159.03ms
step:1264/1480 train_time:199427ms step_avg:159.03ms
step:1265/1480 train_time:199597ms step_avg:159.04ms
step:1266/1480 train_time:199768ms step_avg:159.05ms
step:1267/1480 train_time:199937ms step_avg:159.06ms
step:1268/1480 train_time:200108ms step_avg:159.07ms
step:1269/1480 train_time:200285ms step_avg:159.08ms
step:1270/1480 train_time:200455ms step_avg:159.09ms
step:1271/1480 train_time:200623ms step_avg:159.10ms
step:1272/1480 train_time:200790ms step_avg:159.10ms
step:1273/1480 train_time:200961ms step_avg:159.11ms
step:1274/1480 train_time:201134ms step_avg:159.13ms
step:1275/1480 train_time:201301ms step_avg:159.13ms
step:1276/1480 train_time:201467ms step_avg:159.14ms
step:1277/1480 train_time:201637ms step_avg:159.15ms
step:1278/1480 train_time:201806ms step_avg:159.15ms
step:1279/1480 train_time:201979ms step_avg:159.16ms
step:1280/1480 train_time:202159ms step_avg:159.18ms
step:1281/1480 train_time:202328ms step_avg:159.19ms
step:1282/1480 train_time:202495ms step_avg:159.19ms
step:1283/1480 train_time:202664ms step_avg:159.20ms
step:1284/1480 train_time:202834ms step_avg:159.21ms
step:1285/1480 train_time:203003ms step_avg:159.22ms
step:1286/1480 train_time:203173ms step_avg:159.23ms
step:1287/1480 train_time:203343ms step_avg:159.24ms
step:1288/1480 train_time:203516ms step_avg:159.25ms
step:1289/1480 train_time:203701ms step_avg:159.27ms
step:1290/1480 train_time:203881ms step_avg:159.28ms
step:1291/1480 train_time:204054ms step_avg:159.29ms
step:1292/1480 train_time:204229ms step_avg:159.30ms
step:1293/1480 train_time:204403ms step_avg:159.32ms
step:1294/1480 train_time:204576ms step_avg:159.33ms
step:1295/1480 train_time:204748ms step_avg:159.34ms
step:1296/1480 train_time:204922ms step_avg:159.35ms
step:1297/1480 train_time:205095ms step_avg:159.36ms
step:1298/1480 train_time:205265ms step_avg:159.37ms
step:1299/1480 train_time:205435ms step_avg:159.38ms
step:1300/1480 train_time:205603ms step_avg:159.38ms
step:1301/1480 train_time:205771ms step_avg:159.39ms
step:1302/1480 train_time:205946ms step_avg:159.40ms
step:1303/1480 train_time:206122ms step_avg:159.41ms
step:1304/1480 train_time:206298ms step_avg:159.43ms
step:1305/1480 train_time:206467ms step_avg:159.43ms
step:1306/1480 train_time:206642ms step_avg:159.45ms
step:1307/1480 train_time:206809ms step_avg:159.45ms
step:1308/1480 train_time:206979ms step_avg:159.46ms
step:1309/1480 train_time:207152ms step_avg:159.47ms
step:1310/1480 train_time:207320ms step_avg:159.48ms
step:1311/1480 train_time:207488ms step_avg:159.48ms
step:1312/1480 train_time:207661ms step_avg:159.49ms
step:1313/1480 train_time:207830ms step_avg:159.50ms
step:1314/1480 train_time:208003ms step_avg:159.51ms
step:1315/1480 train_time:208174ms step_avg:159.52ms
step:1316/1480 train_time:208342ms step_avg:159.53ms
step:1317/1480 train_time:208513ms step_avg:159.54ms
step:1318/1480 train_time:208694ms step_avg:159.55ms
step:1319/1480 train_time:208869ms step_avg:159.56ms
step:1320/1480 train_time:209046ms step_avg:159.58ms
step:1321/1480 train_time:209219ms step_avg:159.59ms
step:1322/1480 train_time:209401ms step_avg:159.60ms
step:1323/1480 train_time:209574ms step_avg:159.61ms
step:1324/1480 train_time:209748ms step_avg:159.63ms
step:1325/1480 train_time:209932ms step_avg:159.64ms
step:1326/1480 train_time:210106ms step_avg:159.66ms
step:1327/1480 train_time:210277ms step_avg:159.66ms
step:1328/1480 train_time:210446ms step_avg:159.67ms
step:1329/1480 train_time:210643ms step_avg:159.70ms
step:1330/1480 train_time:210822ms step_avg:159.71ms
step:1331/1480 train_time:210993ms step_avg:159.72ms
step:1332/1480 train_time:211166ms step_avg:159.73ms
step:1333/1480 train_time:211341ms step_avg:159.74ms
step:1334/1480 train_time:211513ms step_avg:159.75ms
step:1335/1480 train_time:211682ms step_avg:159.76ms
step:1336/1480 train_time:211864ms step_avg:159.78ms
step:1337/1480 train_time:212038ms step_avg:159.79ms
step:1338/1480 train_time:212209ms step_avg:159.80ms
step:1339/1480 train_time:212385ms step_avg:159.81ms
step:1340/1480 train_time:212557ms step_avg:159.82ms
step:1341/1480 train_time:212724ms step_avg:159.82ms
step:1342/1480 train_time:212898ms step_avg:159.83ms
step:1343/1480 train_time:213067ms step_avg:159.84ms
step:1344/1480 train_time:213238ms step_avg:159.85ms
step:1345/1480 train_time:213414ms step_avg:159.86ms
step:1346/1480 train_time:213583ms step_avg:159.87ms
step:1347/1480 train_time:213753ms step_avg:159.87ms
step:1348/1480 train_time:213923ms step_avg:159.88ms
step:1349/1480 train_time:214093ms step_avg:159.89ms
step:1350/1480 train_time:214266ms step_avg:159.90ms
step:1351/1480 train_time:214436ms step_avg:159.91ms
step:1352/1480 train_time:214607ms step_avg:159.92ms
step:1353/1480 train_time:214784ms step_avg:159.93ms
step:1354/1480 train_time:214956ms step_avg:159.94ms
step:1355/1480 train_time:215123ms step_avg:159.94ms
step:1356/1480 train_time:215296ms step_avg:159.95ms
step:1357/1480 train_time:215470ms step_avg:159.96ms
step:1358/1480 train_time:215641ms step_avg:159.97ms
step:1359/1480 train_time:215814ms step_avg:159.98ms
step:1360/1480 train_time:215988ms step_avg:159.99ms
step:1361/1480 train_time:216165ms step_avg:160.00ms
step:1362/1480 train_time:216341ms step_avg:160.02ms
step:1363/1480 train_time:216522ms step_avg:160.03ms
step:1364/1480 train_time:216691ms step_avg:160.04ms
step:1365/1480 train_time:216859ms step_avg:160.04ms
step:1366/1480 train_time:217031ms step_avg:160.05ms
step:1367/1480 train_time:217202ms step_avg:160.06ms
step:1368/1480 train_time:217375ms step_avg:160.07ms
step:1369/1480 train_time:217556ms step_avg:160.09ms
step:1370/1480 train_time:217734ms step_avg:160.10ms
step:1371/1480 train_time:217905ms step_avg:160.11ms
step:1372/1480 train_time:218083ms step_avg:160.12ms
step:1373/1480 train_time:218252ms step_avg:160.13ms
step:1374/1480 train_time:218426ms step_avg:160.14ms
step:1375/1480 train_time:218598ms step_avg:160.15ms
step:1375/1480 val_loss:3.2957 train_time:218666ms step_avg:160.19ms
step:1376/1480 train_time:218771ms step_avg:160.15ms
step:1377/1480 train_time:218944ms step_avg:160.16ms
step:1378/1480 train_time:219114ms step_avg:160.17ms
step:1379/1480 train_time:219289ms step_avg:160.18ms
step:1380/1480 train_time:219462ms step_avg:160.19ms
step:1381/1480 train_time:219643ms step_avg:160.21ms
step:1382/1480 train_time:219815ms step_avg:160.22ms
step:1383/1480 train_time:219987ms step_avg:160.22ms
step:1384/1480 train_time:220163ms step_avg:160.24ms
step:1385/1480 train_time:220329ms step_avg:160.24ms
step:1386/1480 train_time:220501ms step_avg:160.25ms
step:1387/1480 train_time:220672ms step_avg:160.26ms
step:1388/1480 train_time:220841ms step_avg:160.26ms
step:1389/1480 train_time:221015ms step_avg:160.27ms
step:1390/1480 train_time:221184ms step_avg:160.28ms
step:1391/1480 train_time:221354ms step_avg:160.29ms
step:1392/1480 train_time:221525ms step_avg:160.29ms
step:1393/1480 train_time:221697ms step_avg:160.30ms
step:1394/1480 train_time:221866ms step_avg:160.31ms
step:1395/1480 train_time:222036ms step_avg:160.31ms
step:1396/1480 train_time:222204ms step_avg:160.32ms
step:1397/1480 train_time:222371ms step_avg:160.33ms
step:1398/1480 train_time:222539ms step_avg:160.33ms
step:1399/1480 train_time:222707ms step_avg:160.34ms
step:1400/1480 train_time:222883ms step_avg:160.35ms
step:1401/1480 train_time:223049ms step_avg:160.35ms
step:1402/1480 train_time:223219ms step_avg:160.36ms
step:1403/1480 train_time:223397ms step_avg:160.37ms
step:1404/1480 train_time:223568ms step_avg:160.38ms
step:1405/1480 train_time:223742ms step_avg:160.39ms
step:1406/1480 train_time:223917ms step_avg:160.40ms
step:1407/1480 train_time:224084ms step_avg:160.40ms
step:1408/1480 train_time:224253ms step_avg:160.41ms
step:1409/1480 train_time:224437ms step_avg:160.43ms
step:1410/1480 train_time:224606ms step_avg:160.43ms
step:1411/1480 train_time:224775ms step_avg:160.44ms
step:1412/1480 train_time:224946ms step_avg:160.45ms
step:1413/1480 train_time:225118ms step_avg:160.45ms
step:1414/1480 train_time:225289ms step_avg:160.46ms
step:1415/1480 train_time:225462ms step_avg:160.47ms
step:1416/1480 train_time:225648ms step_avg:160.49ms
step:1417/1480 train_time:225821ms step_avg:160.50ms
step:1418/1480 train_time:225994ms step_avg:160.51ms
step:1419/1480 train_time:226167ms step_avg:160.52ms
step:1420/1480 train_time:226343ms step_avg:160.53ms
step:1421/1480 train_time:226518ms step_avg:160.54ms
step:1422/1480 train_time:226691ms step_avg:160.55ms
step:1423/1480 train_time:226860ms step_avg:160.55ms
step:1424/1480 train_time:227038ms step_avg:160.56ms
step:1425/1480 train_time:227220ms step_avg:160.58ms
step:1426/1480 train_time:227392ms step_avg:160.59ms
step:1427/1480 train_time:227567ms step_avg:160.60ms
step:1428/1480 train_time:227737ms step_avg:160.60ms
step:1429/1480 train_time:227905ms step_avg:160.61ms
step:1430/1480 train_time:228078ms step_avg:160.62ms
step:1431/1480 train_time:228254ms step_avg:160.63ms
step:1432/1480 train_time:228429ms step_avg:160.64ms
step:1433/1480 train_time:228610ms step_avg:160.65ms
step:1434/1480 train_time:228790ms step_avg:160.67ms
step:1435/1480 train_time:228964ms step_avg:160.68ms
step:1436/1480 train_time:229139ms step_avg:160.69ms
step:1437/1480 train_time:229309ms step_avg:160.69ms
step:1438/1480 train_time:229478ms step_avg:160.70ms
step:1439/1480 train_time:229650ms step_avg:160.71ms
step:1440/1480 train_time:229819ms step_avg:160.71ms
step:1441/1480 train_time:229990ms step_avg:160.72ms
step:1442/1480 train_time:230167ms step_avg:160.73ms
step:1443/1480 train_time:230358ms step_avg:160.75ms
step:1444/1480 train_time:230529ms step_avg:160.76ms
step:1445/1480 train_time:230701ms step_avg:160.77ms
step:1446/1480 train_time:230877ms step_avg:160.78ms
step:1447/1480 train_time:231054ms step_avg:160.79ms
step:1448/1480 train_time:231224ms step_avg:160.80ms
step:1449/1480 train_time:231397ms step_avg:160.80ms
step:1450/1480 train_time:231570ms step_avg:160.81ms
step:1451/1480 train_time:231740ms step_avg:160.82ms
step:1452/1480 train_time:231915ms step_avg:160.83ms
step:1453/1480 train_time:232083ms step_avg:160.83ms
step:1454/1480 train_time:232255ms step_avg:160.84ms
step:1455/1480 train_time:232432ms step_avg:160.85ms
step:1456/1480 train_time:232605ms step_avg:160.86ms
step:1457/1480 train_time:232777ms step_avg:160.87ms
step:1458/1480 train_time:232947ms step_avg:160.88ms
step:1459/1480 train_time:233124ms step_avg:160.89ms
step:1460/1480 train_time:233298ms step_avg:160.89ms
step:1461/1480 train_time:233473ms step_avg:160.91ms
step:1462/1480 train_time:233643ms step_avg:160.91ms
step:1463/1480 train_time:233820ms step_avg:160.92ms
step:1464/1480 train_time:233994ms step_avg:160.93ms
step:1465/1480 train_time:234165ms step_avg:160.94ms
step:1466/1480 train_time:234336ms step_avg:160.95ms
step:1467/1480 train_time:234511ms step_avg:160.95ms
step:1468/1480 train_time:234680ms step_avg:160.96ms
step:1469/1480 train_time:234853ms step_avg:160.97ms
step:1470/1480 train_time:235033ms step_avg:160.98ms
step:1471/1480 train_time:235220ms step_avg:161.00ms
step:1472/1480 train_time:235401ms step_avg:161.01ms
step:1473/1480 train_time:235572ms step_avg:161.02ms
step:1474/1480 train_time:235751ms step_avg:161.03ms
step:1475/1480 train_time:235930ms step_avg:161.04ms
step:1476/1480 train_time:236102ms step_avg:161.05ms
step:1477/1480 train_time:236283ms step_avg:161.07ms
step:1478/1480 train_time:236465ms step_avg:161.08ms
step:1479/1480 train_time:236640ms step_avg:161.09ms
step:1480/1480 train_time:236813ms step_avg:161.10ms
step:1480/1480 val_loss:3.2768 train_time:236884ms step_avg:161.15ms