625a6fcc-203c-4545-b697-0b8daa2b6d07.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
from dataclasses import dataclass
from pathlib import Path

import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
# Use of FlexAttention contributed by @KoszarskyB
from torch.nn.attention.flex_attention import BlockMask, flex_attention

# -----------------------------------------------------------------------------
# Muon optimizer

def zeropower_via_svd(G, steps=None):
    U, S, V = G.svd()
    return U @ V.T

@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert len(G.shape) == 2
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    X /= (X.norm() + eps) # ensure top singular value <= 1
    if G.size(0) > G.size(1):
        X = X.T
    for _ in range(steps):
        A = X @ X.T
        B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    if G.size(0) > G.size(1):
        X = X.T
    return X

zeropower_backends = dict(svd=zeropower_via_svd, newtonschulz5=zeropower_via_newtonschulz5)

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven't tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        backend: The chosen backend for the orthogonalization step. (recommended: 'newtonschulz5')
        backend_steps: The number of iteration steps to use in the backend, if it is iterative.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True,
                 backend='newtonschulz5', backend_steps=5):
        self.num_process = int(os.environ['WORLD_SIZE'])
        self.rank = int(os.environ["RANK"])
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, backend=backend, backend_steps=backend_steps)
        params: "list[torch.Tensor]" = list(params)
        assert all(isinstance(p, torch.Tensor) for p in params)
        sizes = {p.numel() for p in params}
        param_groups = [
            {
                "params": [p for p in params if p.numel() == size],
                "update_buffer": [
                    torch.empty(size, device="cuda", dtype=torch.bfloat16)
                    for _ in range(self.num_process)
                ],
            }
            for size in sizes
        ]
        super().__init__(param_groups, defaults)

    def step(self):
        for group in self.param_groups:
            lr: float = group["lr"]
            momentum: float = group["momentum"]
            nesterov: bool = group["nesterov"]
            zeropower_backend = zeropower_backends[group["backend"]]
            backend_steps: int = group["backend_steps"]
            update_buffers: "list[torch.Tensor]" = group["update_buffer"]
            # generate weight updates in distributed fashion
            params: "list[torch.Tensor]" = group["params"]
            assert len(params) % self.num_process == 0
            handle = None
            params_world = None
            def update_prev():
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffers):
                    p_world.data.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.num_process]:
                p = params[base_i + self.rank]
                g = p.grad
                assert g is not None
                state = self.state[p] 
                if "momentum_buffer" not in state:
                    state["momentum_buffer"] = torch.zeros_like(g)
                buf: torch.Tensor = state["momentum_buffer"]
                buf.lerp_(g, 1 - momentum)
                g = g.lerp_(buf, momentum) if nesterov else buf
                g = zeropower_backend(g, steps=backend_steps).flatten()
                update_prev()
                handle = dist.all_gather(update_buffers, g, async_op=True)
                params_world = params[base_i : base_i + self.num_process]
            update_prev()


# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):

    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=False)

    def forward(self, x):
        return F.linear(x, self.weight.to(x.dtype))

class Rotary(torch.nn.Module):

    def __init__(self, dim, base=10000):
        super().__init__()
        self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None

    def forward(self, x):
        seq_len = x.shape[1]
        if seq_len != self.seq_len_cached:
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.outer(t, self.inv_freq)
            self.seq_len_cached = seq_len
            self.cos_cached = freqs.cos()
            self.sin_cached = freqs.sin()
        cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
        # apply_rotary_emb(x, cos, sin)
        x1, x2 = x.chunk(2, dim=3)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x)

class CausalSelfAttention(nn.Module):

    def __init__(self, dim, n_head):
        super().__init__()
        assert dim % n_head == 0
        self.n_head = n_head
        self.c_q = CastedLinear(dim, dim)
        self.c_k = CastedLinear(dim, dim)
        self.c_v = CastedLinear(dim, dim)
        # value residual lambda
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5])) # @Grad62304977
        # rotary embeddings
        self.rotary = Rotary(dim // n_head) # dim // n_head = head_dim
        # output projection
        self.c_proj = CastedLinear(dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor, vi: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q: torch.Tensor = self.c_q(x).view(B, T, self.n_head, -1)
        k: torch.Tensor = self.c_k(x).view(B, T, self.n_head, -1)
        v: torch.Tensor = self.c_v(x).view(B, T, self.n_head, -1)
        v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @Grad62304977
        q, k = norm(q), norm(k) # QK norm suggested by @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask)
        y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):

    def __init__(self, dim: int):
        super().__init__()
        self.c_fc   = CastedLinear(dim, 4 * dim)
        self.c_proj = CastedLinear(4 * dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):

    def __init__(self, config):
        super().__init__()
        self.attn = CausalSelfAttention(config.n_embd, config.n_head)
        self.mlp = MLP(config.n_embd)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x: torch.Tensor, vi: torch.Tensor, x0: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        x = x + self.attn(norm(x), vi, block_mask)
        x = x + self.mlp(norm(x))
        return x

# -----------------------------------------------------------------------------
# The main GPT-2 model

@dataclass
class GPTConfig:
    vocab_size : int = 50304
    n_layer : int = 12
    n_head : int = 6 # head dim 128 suggested by @Grad62304977
    n_embd : int = 768
    lm_head_softcap : int = 30

class GPT(nn.Module):

    def __init__(self, config: GPTConfig):
        super().__init__()
        self.n_layer = config.n_layer
        self.lm_head_softcap = config.lm_head_softcap

        # U-net design by @brendanh0gan
        self.num_encoder_layers = config.n_layer // 2 # Half of the layers for encoder
        self.num_decoder_layers = config.n_layer - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))

        self.transformer = nn.ModuleDict(dict(
            wte = nn.Embedding(config.vocab_size, config.n_embd),
            # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
            # U-net structure on token value embeddings by @leloykun
            vte = nn.Embedding(config.vocab_size, config.n_embd*self.num_encoder_layers),
            h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
        ))
        self.lm_head = CastedLinear(config.n_embd, config.vocab_size)
        self.lm_head.weight.data.zero_() # @Grad62304977

    def forward(self, idx: torch.Tensor, target: torch.Tensor, sliding_window: torch.Tensor) -> torch.Tensor:
        BLOCK_SIZE = 128
        assert idx.ndim == 1
        docs = (idx == 50256).cumsum(0)
        docs_low = docs.reshape(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.reshape(-1, BLOCK_SIZE)[:, -1].contiguous()
        def document_sliding_window_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            window_mask = q_idx - kv_idx < sliding_window
            return causal_mask & document_mask & window_mask

        S = len(idx)
        def create_sliding_window_causal_mask(S: int, sliding_window: torch.Tensor):
            kv_idx = block_idx = torch.arange(S // BLOCK_SIZE, dtype=torch.int32, device="cuda")
            q_idx = block_idx[:, None]
            causal_mask = q_idx >= kv_idx
            document_mask = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
            window_mask = q_idx - kv_idx < ((sliding_window + BLOCK_SIZE - 1) // BLOCK_SIZE)
            dense_mask = causal_mask & document_mask & window_mask
            dense_mask = dense_mask.to(torch.int32)
            num_blocks = dense_mask.sum(dim=-1).to(torch.int32)
            indices = torch.argsort(dense_mask, dim=-1, descending=True, stable=True).to(torch.int32)
            num_blocks = num_blocks[None, None, :].contiguous()
            indices = indices[None, None, :].contiguous()
            return BlockMask.from_kv_blocks(num_blocks, indices, BLOCK_SIZE=BLOCK_SIZE, mask_mod=document_sliding_window_causal)
        block_mask = create_sliding_window_causal_mask(S, sliding_window)

        # forward the GPT model itself
        x = self.transformer.wte(idx[None]) # token embeddings of shape (b, t, n_embd)
        x = norm(x) # @Grad62304977
        x0 = x
        vi = self.transformer.vte(idx[None]).chunk(self.num_encoder_layers, dim=-1)

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        for i in range(self.num_encoder_layers):
            x = self.transformer.h[i](x, vi[i], x0, block_mask)
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            # U-net structure on token value embeddings by @leloykun
            x = self.transformer.h[self.num_encoder_layers + i](x, vi[self.num_encoder_layers-1-i], x0, block_mask)

        x = norm(x)
        logits = self.lm_head(x)
        logits = self.lm_head_softcap * torch.tanh(logits / self.lm_head_softcap) # @Grad62304977
        logits = logits.float()
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), target.view(-1))
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _peek_data_shard(file: Path):
    # only reads the header, returns header data
    # header is 256 int32
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    return int(header[2]) # number of tokens (claimed)

def _load_data_shard(file: Path, ntok: int):
    with file.open("rb") as f:
        tokens = torch.empty(ntok, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy())
        assert nbytes == 2 * ntok, "number of tokens read does not match header?"
    return tokens

class DistributedDataLoader:
    def __init__(self, filename_pattern, T, process_rank, num_processes):
        self.process_rank = process_rank
        self.num_processes = num_processes
        self.T = T

        # glob files that match the pattern
        self.files = sorted(Path.cwd().glob(filename_pattern))
        assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"

        # load and validate all data shards, count number of tokens in total
        self.ntoks = [_peek_data_shard(file) for file in self.files]
        assert min(self.ntoks) >= num_processes * T + 1
        self.ntok_total = sum(self.ntoks)

        self.reset()

    def reset(self):
        self.current_shard = -1
        self.advance()

    def advance(self): # advance to next data shard
        self.current_shard = (self.current_shard + 1) % len(self.files)
        self.current_position = self.process_rank * self.T
        self.tokens = _load_data_shard(self.files[self.current_shard], self.ntoks[self.current_shard])

    def next_batch(self):
        batch_size = self.T * self.num_processes
        buf = self.tokens[self.current_position:self.current_position+self.T+1]
        # host side async is sufficient;
        # no performance improvement was observed when introducing a separate stream.
        x = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
        y = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
        # advance current position and load next shard if necessary
        self.current_position += batch_size
        if self.current_position + batch_size + 1 >= len(self.tokens):
            self.advance()
        return x, y

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data hyperparams
    input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
    input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
    # optimization hyperparams
    batch_size : int = 8 # batch size, in sequences, across all devices
    sequence_length : int = 64*1024 # sequence length, in tokens
    num_iterations : int = 1480 # number of iterations to run
    warmup_iters : int = 0
    cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
    weight_decay : float = 0
    # evaluation and logging hyperparams
    val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
    val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()

# set up DDP (distributed data parallel). torchrun sets this env variable
assert torch.cuda.is_available()
dist.init_process_group(backend='nccl')
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
device = f'cuda:{ddp_local_rank}'
torch.cuda.set_device(device)
print(f"using device: {device}")
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = str(uuid.uuid4())
    logdir = 'logs/%s/' % run_id
    # os.makedirs(logdir, exist_ok=True)
    logfile = 'logs/%s.txt' % run_id
    # create the log file
    with open(logfile, "w") as f:
        # begin the log by printing this file (the Python code)
        f.write(code)
        f.write('='*100 + '\n')
def print0(s, logonly=False):
    if master_process:
        with open(logfile, "a") as f:
            if not logonly:
                print(s)
            f.write(s+'\n')
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)

# convenience variables
T = args.sequence_length
# calculate the number of steps to take in the val loop.
assert args.val_tokens % (T * ddp_world_size) == 0
val_steps = args.val_tokens // (T * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size
assert train_accumulation_steps == 1

# load tokens
train_loader = DistributedDataLoader(args.input_bin, T, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, T, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.ntok_total} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.ntok_total} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
x, y = train_loader.next_batch()

# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, n_layer=12, n_head=6, n_embd=768))
model = model.cuda().bfloat16()
for m in model.modules():
    if isinstance(m, CastedLinear):
        m.float()
if hasattr(config, "coordinate_descent_tuning"):
    config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank])
raw_model = model.module # always contains the "raw" unwrapped model

# init the optimizer(s)
optimizer1 = torch.optim.Adam([raw_model.transformer.wte.weight, raw_model.transformer.vte.weight], lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.transformer.h.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
    assert it <= args.num_iterations
    # 1) linear warmup for warmup_iters steps
    if it < args.warmup_iters:
        return (it+1) / args.warmup_iters
    # 2) constant lr for a while
    elif it < args.num_iterations - args.cooldown_iters:
        return 1.0
    # 3) linear cooldown
    else:
        decay_ratio = (args.num_iterations - it) / args.cooldown_iters
        return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]

sliding_window_size = torch.tensor(64, dtype=torch.int32, device="cuda")
sw_size_prev = 64
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
    last_step = (step == args.num_iterations)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Set the sliding window size for the current step, in chunks of 64. By @fernbear.bsky.social
    sw_size =  64 * int((64 + (1792 - 64) * step / args.num_iterations) // 64)
    if sw_size != sw_size_prev:
        sliding_window_size.copy_(sw_size, non_blocking=True)
        sw_size_prev = sw_size

    # once in a while evaluate the validation dataset
    if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # run validation batches
        model.eval()
        val_loader.reset()
        val_loss = 0.0
        for _ in range(val_steps):
            with torch.no_grad():
                x_val, y_val = val_loader.next_batch()
                val_loss += model(x_val, y_val, sliding_window=sliding_window_size)
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        val_loss /= val_steps
        # log val loss to console and to logfile
        print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # save the state of the training process
        log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
        # torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    # bit confusing: we want to make sure to eval on 0th iteration
    # but also after the very last iteration. so we loop for step <= num_iterations
    # instead of just < num_iterations (one extra due to <=), only to do
    # the validation/sampling one last time, and then we break right here as we're done.
    if last_step:
        break

    # --------------- TRAINING SECTION BEGIN -----------------
    model.train()
    loss = model(x, y, sliding_window=sliding_window_size)
    loss.backward()
    del loss
    # advance the dataset for the next batch
    x, y = train_loader.next_batch()
    # momentum warmup for Muon
    frac = min(step/300, 1)
    for group in optimizer3.param_groups:
        group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # --------------- TRAINING SECTION END -------------------
    # everything that follows now is just diagnostics, prints, logging, etc.
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")

if master_process:
    print(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")

# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()
====================================================================================================
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Sun Dec  8 10:15:24 2024       
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.6     |
|-----------------------------------------+----------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
|                                         |                      |               MIG M. |
|=========================================+======================+======================|
|   0  NVIDIA H100 80GB HBM3          On  | 00000000:65:02.0 Off |                    0 |
| N/A   36C    P0              74W / 700W |      7MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  | 00000000:67:02.0 Off |                    0 |
| N/A   46C    P0             131W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  | 00000000:69:02.0 Off |                    0 |
| N/A   45C    P0              83W / 700W |     26MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  | 00000000:6B:02.0 Off |                    0 |
| N/A   39C    P0             118W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  | 00000000:6F:02.0 Off |                    0 |
| N/A   39C    P0             117W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  | 00000000:71:02.0 Off |                    0 |
| N/A   45C    P0              93W / 700W |     26MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  | 00000000:73:02.0 Off |                    0 |
| N/A   46C    P0             127W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  | 00000000:75:02.0 Off |                    0 |
| N/A   38C    P0             115W / 700W |     45MiB / 81559MiB |      1%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
                                                                                         
+---------------------------------------------------------------------------------------+
| Processes:                                                                            |
|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
|        ID   ID                                                             Usage      |
|=======================================================================================|
+---------------------------------------------------------------------------------------+

====================================================================================================
Training DataLoader: total number of tokens: 3200000000 across 32 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:22986ms step_avg:nanms
step:2/1480 train_time:23073ms step_avg:nanms
step:3/1480 train_time:23210ms step_avg:nanms
step:4/1480 train_time:23350ms step_avg:nanms
step:5/1480 train_time:23492ms step_avg:nanms
step:6/1480 train_time:23633ms step_avg:nanms
step:7/1480 train_time:23774ms step_avg:nanms
step:8/1480 train_time:23917ms step_avg:nanms
step:9/1480 train_time:24063ms step_avg:nanms
step:10/1480 train_time:24207ms step_avg:nanms
step:11/1480 train_time:140ms step_avg:nanms
step:12/1480 train_time:281ms step_avg:nanms
step:13/1480 train_time:424ms step_avg:141.31ms
step:14/1480 train_time:565ms step_avg:141.32ms
step:15/1480 train_time:707ms step_avg:141.42ms
step:16/1480 train_time:852ms step_avg:142.00ms
step:17/1480 train_time:997ms step_avg:142.49ms
step:18/1480 train_time:1139ms step_avg:142.43ms
step:19/1480 train_time:1280ms step_avg:142.26ms
step:20/1480 train_time:1422ms step_avg:142.20ms
step:21/1480 train_time:1564ms step_avg:142.16ms
step:22/1480 train_time:1708ms step_avg:142.32ms
step:23/1480 train_time:1851ms step_avg:142.39ms
step:24/1480 train_time:1996ms step_avg:142.60ms
step:25/1480 train_time:2139ms step_avg:142.62ms
step:26/1480 train_time:2282ms step_avg:142.60ms
step:27/1480 train_time:2423ms step_avg:142.56ms
step:28/1480 train_time:2566ms step_avg:142.53ms
step:29/1480 train_time:2708ms step_avg:142.55ms
step:30/1480 train_time:2852ms step_avg:142.62ms
step:31/1480 train_time:2998ms step_avg:142.75ms
step:32/1480 train_time:3140ms step_avg:142.74ms
step:33/1480 train_time:3282ms step_avg:142.68ms
step:34/1480 train_time:3424ms step_avg:142.66ms
step:35/1480 train_time:3566ms step_avg:142.62ms
step:36/1480 train_time:3707ms step_avg:142.56ms
step:37/1480 train_time:3848ms step_avg:142.52ms
step:38/1480 train_time:3993ms step_avg:142.59ms
step:39/1480 train_time:4136ms step_avg:142.63ms
step:40/1480 train_time:4279ms step_avg:142.62ms
step:41/1480 train_time:4422ms step_avg:142.64ms
step:42/1480 train_time:4565ms step_avg:142.65ms
step:43/1480 train_time:4708ms step_avg:142.66ms
step:44/1480 train_time:4852ms step_avg:142.71ms
step:45/1480 train_time:4997ms step_avg:142.77ms
step:46/1480 train_time:5139ms step_avg:142.76ms
step:47/1480 train_time:5281ms step_avg:142.74ms
step:48/1480 train_time:5425ms step_avg:142.76ms
step:49/1480 train_time:5569ms step_avg:142.79ms
step:50/1480 train_time:5712ms step_avg:142.80ms
step:51/1480 train_time:5854ms step_avg:142.79ms
step:52/1480 train_time:5998ms step_avg:142.80ms
step:53/1480 train_time:6140ms step_avg:142.79ms
step:54/1480 train_time:6281ms step_avg:142.76ms
step:55/1480 train_time:6425ms step_avg:142.77ms
step:56/1480 train_time:6567ms step_avg:142.76ms
step:57/1480 train_time:6711ms step_avg:142.79ms
step:58/1480 train_time:6855ms step_avg:142.82ms
step:59/1480 train_time:6999ms step_avg:142.83ms
step:60/1480 train_time:7141ms step_avg:142.82ms
step:61/1480 train_time:7282ms step_avg:142.79ms
step:62/1480 train_time:7425ms step_avg:142.78ms
step:63/1480 train_time:7569ms step_avg:142.82ms
step:64/1480 train_time:7716ms step_avg:142.89ms
step:65/1480 train_time:7860ms step_avg:142.90ms
step:66/1480 train_time:8001ms step_avg:142.88ms
step:67/1480 train_time:8142ms step_avg:142.85ms
step:68/1480 train_time:8286ms step_avg:142.86ms
step:69/1480 train_time:8430ms step_avg:142.88ms
step:70/1480 train_time:8573ms step_avg:142.88ms
step:71/1480 train_time:8717ms step_avg:142.91ms
step:72/1480 train_time:8860ms step_avg:142.90ms
step:73/1480 train_time:9002ms step_avg:142.89ms
step:74/1480 train_time:9144ms step_avg:142.88ms
step:75/1480 train_time:9289ms step_avg:142.90ms
step:76/1480 train_time:9432ms step_avg:142.91ms
step:77/1480 train_time:9575ms step_avg:142.91ms
step:78/1480 train_time:9719ms step_avg:142.93ms
step:79/1480 train_time:9861ms step_avg:142.91ms
step:80/1480 train_time:10001ms step_avg:142.88ms
step:81/1480 train_time:10142ms step_avg:142.85ms
step:82/1480 train_time:10284ms step_avg:142.84ms
step:83/1480 train_time:10428ms step_avg:142.85ms
step:84/1480 train_time:10572ms step_avg:142.86ms
step:85/1480 train_time:10715ms step_avg:142.86ms
step:86/1480 train_time:10857ms step_avg:142.86ms
step:87/1480 train_time:11000ms step_avg:142.86ms
step:88/1480 train_time:11141ms step_avg:142.83ms
step:89/1480 train_time:11282ms step_avg:142.82ms
step:90/1480 train_time:11426ms step_avg:142.83ms
step:91/1480 train_time:11571ms step_avg:142.85ms
step:92/1480 train_time:11715ms step_avg:142.86ms
step:93/1480 train_time:11857ms step_avg:142.86ms
step:94/1480 train_time:12000ms step_avg:142.86ms
step:95/1480 train_time:12141ms step_avg:142.84ms
step:96/1480 train_time:12284ms step_avg:142.84ms
step:97/1480 train_time:12427ms step_avg:142.84ms
step:98/1480 train_time:12572ms step_avg:142.86ms
step:99/1480 train_time:12718ms step_avg:142.90ms
step:100/1480 train_time:12860ms step_avg:142.89ms
step:101/1480 train_time:13001ms step_avg:142.87ms
step:102/1480 train_time:13142ms step_avg:142.85ms
step:103/1480 train_time:13286ms step_avg:142.86ms
step:104/1480 train_time:13431ms step_avg:142.89ms
step:105/1480 train_time:13575ms step_avg:142.90ms
step:106/1480 train_time:13719ms step_avg:142.91ms
step:107/1480 train_time:13862ms step_avg:142.90ms
step:108/1480 train_time:14005ms step_avg:142.91ms
step:109/1480 train_time:14147ms step_avg:142.90ms
step:110/1480 train_time:14291ms step_avg:142.91ms
step:111/1480 train_time:14438ms step_avg:142.95ms
step:112/1480 train_time:14584ms step_avg:142.99ms
step:113/1480 train_time:14733ms step_avg:143.04ms
step:114/1480 train_time:14880ms step_avg:143.08ms
step:115/1480 train_time:15025ms step_avg:143.10ms
step:116/1480 train_time:15173ms step_avg:143.14ms
step:117/1480 train_time:15320ms step_avg:143.18ms
step:118/1480 train_time:15466ms step_avg:143.21ms
step:119/1480 train_time:15615ms step_avg:143.26ms
step:120/1480 train_time:15762ms step_avg:143.29ms
step:121/1480 train_time:15909ms step_avg:143.32ms
step:122/1480 train_time:16056ms step_avg:143.35ms
step:123/1480 train_time:16202ms step_avg:143.38ms
step:124/1480 train_time:16349ms step_avg:143.41ms
step:125/1480 train_time:16498ms step_avg:143.46ms
step:125/1480 val_loss:4.4193 train_time:16555ms step_avg:143.96ms
step:126/1480 train_time:16651ms step_avg:143.54ms
step:127/1480 train_time:16799ms step_avg:143.58ms
step:128/1480 train_time:16947ms step_avg:143.62ms
step:129/1480 train_time:17092ms step_avg:143.63ms
step:130/1480 train_time:17238ms step_avg:143.65ms
step:131/1480 train_time:17385ms step_avg:143.68ms
step:132/1480 train_time:17532ms step_avg:143.70ms
step:133/1480 train_time:17680ms step_avg:143.74ms
step:134/1480 train_time:17829ms step_avg:143.78ms
step:135/1480 train_time:17977ms step_avg:143.81ms
step:136/1480 train_time:18125ms step_avg:143.85ms
step:137/1480 train_time:18271ms step_avg:143.86ms
step:138/1480 train_time:18417ms step_avg:143.88ms
step:139/1480 train_time:18565ms step_avg:143.91ms
step:140/1480 train_time:18711ms step_avg:143.93ms
step:141/1480 train_time:18859ms step_avg:143.96ms
step:142/1480 train_time:19007ms step_avg:143.99ms
step:143/1480 train_time:19154ms step_avg:144.01ms
step:144/1480 train_time:19302ms step_avg:144.05ms
step:145/1480 train_time:19450ms step_avg:144.07ms
step:146/1480 train_time:19595ms step_avg:144.08ms
step:147/1480 train_time:19742ms step_avg:144.10ms
step:148/1480 train_time:19889ms step_avg:144.12ms
step:149/1480 train_time:20034ms step_avg:144.13ms
step:150/1480 train_time:20182ms step_avg:144.16ms
step:151/1480 train_time:20330ms step_avg:144.18ms
step:152/1480 train_time:20476ms step_avg:144.20ms
step:153/1480 train_time:20625ms step_avg:144.23ms
step:154/1480 train_time:20771ms step_avg:144.24ms
step:155/1480 train_time:20918ms step_avg:144.26ms
step:156/1480 train_time:21066ms step_avg:144.29ms
step:157/1480 train_time:21211ms step_avg:144.29ms
step:158/1480 train_time:21359ms step_avg:144.32ms
step:159/1480 train_time:21506ms step_avg:144.34ms
step:160/1480 train_time:21653ms step_avg:144.35ms
step:161/1480 train_time:21802ms step_avg:144.38ms
step:162/1480 train_time:21949ms step_avg:144.40ms
step:163/1480 train_time:22096ms step_avg:144.42ms
step:164/1480 train_time:22244ms step_avg:144.44ms
step:165/1480 train_time:22390ms step_avg:144.45ms
step:166/1480 train_time:22537ms step_avg:144.47ms
step:167/1480 train_time:22684ms step_avg:144.48ms
step:168/1480 train_time:22831ms step_avg:144.50ms
step:169/1480 train_time:22978ms step_avg:144.52ms
step:170/1480 train_time:23126ms step_avg:144.54ms
step:171/1480 train_time:23272ms step_avg:144.55ms
step:172/1480 train_time:23418ms step_avg:144.55ms
step:173/1480 train_time:23565ms step_avg:144.57ms
step:174/1480 train_time:23711ms step_avg:144.58ms
step:175/1480 train_time:23859ms step_avg:144.60ms
step:176/1480 train_time:24007ms step_avg:144.62ms
step:177/1480 train_time:24154ms step_avg:144.64ms
step:178/1480 train_time:24303ms step_avg:144.66ms
step:179/1480 train_time:24450ms step_avg:144.67ms
step:180/1480 train_time:24595ms step_avg:144.68ms
step:181/1480 train_time:24744ms step_avg:144.70ms
step:182/1480 train_time:24891ms step_avg:144.71ms
step:183/1480 train_time:25037ms step_avg:144.72ms
step:184/1480 train_time:25185ms step_avg:144.74ms
step:185/1480 train_time:25332ms step_avg:144.75ms
step:186/1480 train_time:25478ms step_avg:144.76ms
step:187/1480 train_time:25625ms step_avg:144.77ms
step:188/1480 train_time:25772ms step_avg:144.79ms
step:189/1480 train_time:25921ms step_avg:144.81ms
step:190/1480 train_time:26069ms step_avg:144.83ms
step:191/1480 train_time:26215ms step_avg:144.83ms
step:192/1480 train_time:26363ms step_avg:144.85ms
step:193/1480 train_time:26509ms step_avg:144.86ms
step:194/1480 train_time:26656ms step_avg:144.87ms
step:195/1480 train_time:26804ms step_avg:144.89ms
step:196/1480 train_time:26951ms step_avg:144.90ms
step:197/1480 train_time:27099ms step_avg:144.91ms
step:198/1480 train_time:27247ms step_avg:144.93ms
step:199/1480 train_time:27392ms step_avg:144.93ms
step:200/1480 train_time:27541ms step_avg:144.95ms
step:201/1480 train_time:27688ms step_avg:144.96ms
step:202/1480 train_time:27835ms step_avg:144.98ms
step:203/1480 train_time:27983ms step_avg:144.99ms
step:204/1480 train_time:28130ms step_avg:145.00ms
step:205/1480 train_time:28276ms step_avg:145.01ms
step:206/1480 train_time:28424ms step_avg:145.02ms
step:207/1480 train_time:28570ms step_avg:145.03ms
step:208/1480 train_time:28718ms step_avg:145.04ms
step:209/1480 train_time:28865ms step_avg:145.05ms
step:210/1480 train_time:29011ms step_avg:145.05ms
step:211/1480 train_time:29158ms step_avg:145.07ms
step:212/1480 train_time:29306ms step_avg:145.08ms
step:213/1480 train_time:29453ms step_avg:145.09ms
step:214/1480 train_time:29600ms step_avg:145.10ms
step:215/1480 train_time:29748ms step_avg:145.11ms
step:216/1480 train_time:29893ms step_avg:145.11ms
step:217/1480 train_time:30040ms step_avg:145.12ms
step:218/1480 train_time:30187ms step_avg:145.13ms
step:219/1480 train_time:30334ms step_avg:145.14ms
step:220/1480 train_time:30481ms step_avg:145.15ms
step:221/1480 train_time:30631ms step_avg:145.17ms
step:222/1480 train_time:30783ms step_avg:145.20ms
step:223/1480 train_time:30934ms step_avg:145.23ms
step:224/1480 train_time:31084ms step_avg:145.25ms
step:225/1480 train_time:31235ms step_avg:145.28ms
step:226/1480 train_time:31385ms step_avg:145.30ms
step:227/1480 train_time:31535ms step_avg:145.32ms
step:228/1480 train_time:31685ms step_avg:145.35ms
step:229/1480 train_time:31838ms step_avg:145.38ms
step:230/1480 train_time:31989ms step_avg:145.40ms
step:231/1480 train_time:32139ms step_avg:145.42ms
step:232/1480 train_time:32289ms step_avg:145.45ms
step:233/1480 train_time:32440ms step_avg:145.47ms
step:234/1480 train_time:32591ms step_avg:145.49ms
step:235/1480 train_time:32742ms step_avg:145.52ms
step:236/1480 train_time:32893ms step_avg:145.54ms
step:237/1480 train_time:33044ms step_avg:145.57ms
step:238/1480 train_time:33194ms step_avg:145.59ms
step:239/1480 train_time:33343ms step_avg:145.60ms
step:240/1480 train_time:33493ms step_avg:145.62ms
step:241/1480 train_time:33644ms step_avg:145.65ms
step:242/1480 train_time:33794ms step_avg:145.66ms
step:243/1480 train_time:33947ms step_avg:145.70ms
step:244/1480 train_time:34098ms step_avg:145.72ms
step:245/1480 train_time:34249ms step_avg:145.74ms
step:246/1480 train_time:34397ms step_avg:145.75ms
step:247/1480 train_time:34549ms step_avg:145.78ms
step:248/1480 train_time:34700ms step_avg:145.80ms
step:249/1480 train_time:34852ms step_avg:145.82ms
step:250/1480 train_time:35003ms step_avg:145.85ms
step:250/1480 val_loss:3.9961 train_time:35063ms step_avg:146.09ms
step:251/1480 train_time:35161ms step_avg:145.90ms
step:252/1480 train_time:35313ms step_avg:145.92ms
step:253/1480 train_time:35464ms step_avg:145.94ms
step:254/1480 train_time:35615ms step_avg:145.96ms
step:255/1480 train_time:35765ms step_avg:145.98ms
step:256/1480 train_time:35915ms step_avg:146.00ms
step:257/1480 train_time:36065ms step_avg:146.01ms
step:258/1480 train_time:36216ms step_avg:146.03ms
step:259/1480 train_time:36368ms step_avg:146.06ms
step:260/1480 train_time:36520ms step_avg:146.08ms
step:261/1480 train_time:36670ms step_avg:146.10ms
step:262/1480 train_time:36821ms step_avg:146.11ms
step:263/1480 train_time:36970ms step_avg:146.13ms
step:264/1480 train_time:37119ms step_avg:146.14ms
step:265/1480 train_time:37271ms step_avg:146.16ms
step:266/1480 train_time:37421ms step_avg:146.18ms
step:267/1480 train_time:37572ms step_avg:146.19ms
step:268/1480 train_time:37722ms step_avg:146.21ms
step:269/1480 train_time:37874ms step_avg:146.23ms
step:270/1480 train_time:38024ms step_avg:146.25ms
step:271/1480 train_time:38175ms step_avg:146.26ms
step:272/1480 train_time:38325ms step_avg:146.28ms
step:273/1480 train_time:38477ms step_avg:146.30ms
step:274/1480 train_time:38625ms step_avg:146.31ms
step:275/1480 train_time:38778ms step_avg:146.33ms
step:276/1480 train_time:38928ms step_avg:146.35ms
step:277/1480 train_time:39079ms step_avg:146.36ms
step:278/1480 train_time:39228ms step_avg:146.37ms
step:279/1480 train_time:39379ms step_avg:146.39ms
step:280/1480 train_time:39530ms step_avg:146.41ms
step:281/1480 train_time:39681ms step_avg:146.42ms
step:282/1480 train_time:39831ms step_avg:146.44ms
step:283/1480 train_time:39982ms step_avg:146.45ms
step:284/1480 train_time:40131ms step_avg:146.46ms
step:285/1480 train_time:40283ms step_avg:146.48ms
step:286/1480 train_time:40434ms step_avg:146.50ms
step:287/1480 train_time:40584ms step_avg:146.51ms
step:288/1480 train_time:40735ms step_avg:146.53ms
step:289/1480 train_time:40885ms step_avg:146.54ms
step:290/1480 train_time:41035ms step_avg:146.55ms
step:291/1480 train_time:41185ms step_avg:146.57ms
step:292/1480 train_time:41336ms step_avg:146.58ms
step:293/1480 train_time:41486ms step_avg:146.59ms
step:294/1480 train_time:41638ms step_avg:146.61ms
step:295/1480 train_time:41788ms step_avg:146.62ms
step:296/1480 train_time:41939ms step_avg:146.64ms
step:297/1480 train_time:42089ms step_avg:146.65ms
step:298/1480 train_time:42241ms step_avg:146.67ms
step:299/1480 train_time:42391ms step_avg:146.68ms
step:300/1480 train_time:42542ms step_avg:146.70ms
step:301/1480 train_time:42692ms step_avg:146.71ms
step:302/1480 train_time:42842ms step_avg:146.72ms
step:303/1480 train_time:42993ms step_avg:146.73ms
step:304/1480 train_time:43143ms step_avg:146.74ms
step:305/1480 train_time:43292ms step_avg:146.75ms
step:306/1480 train_time:43443ms step_avg:146.77ms
step:307/1480 train_time:43595ms step_avg:146.78ms
step:308/1480 train_time:43746ms step_avg:146.80ms
step:309/1480 train_time:43898ms step_avg:146.81ms
step:310/1480 train_time:44047ms step_avg:146.82ms
step:311/1480 train_time:44198ms step_avg:146.84ms
step:312/1480 train_time:44348ms step_avg:146.85ms
step:313/1480 train_time:44499ms step_avg:146.86ms
step:314/1480 train_time:44648ms step_avg:146.87ms
step:315/1480 train_time:44800ms step_avg:146.88ms
step:316/1480 train_time:44951ms step_avg:146.90ms
step:317/1480 train_time:45102ms step_avg:146.91ms
step:318/1480 train_time:45252ms step_avg:146.92ms
step:319/1480 train_time:45402ms step_avg:146.93ms
step:320/1480 train_time:45553ms step_avg:146.95ms
step:321/1480 train_time:45704ms step_avg:146.96ms
step:322/1480 train_time:45855ms step_avg:146.97ms
step:323/1480 train_time:46004ms step_avg:146.98ms
step:324/1480 train_time:46155ms step_avg:146.99ms
step:325/1480 train_time:46305ms step_avg:147.00ms
step:326/1480 train_time:46457ms step_avg:147.02ms
step:327/1480 train_time:46608ms step_avg:147.03ms
step:328/1480 train_time:46759ms step_avg:147.04ms
step:329/1480 train_time:46909ms step_avg:147.05ms
step:330/1480 train_time:47061ms step_avg:147.07ms
step:331/1480 train_time:47214ms step_avg:147.09ms
step:332/1480 train_time:47369ms step_avg:147.11ms
step:333/1480 train_time:47522ms step_avg:147.13ms
step:334/1480 train_time:47676ms step_avg:147.15ms
step:335/1480 train_time:47829ms step_avg:147.17ms
step:336/1480 train_time:47983ms step_avg:147.19ms
step:337/1480 train_time:48138ms step_avg:147.21ms
step:338/1480 train_time:48291ms step_avg:147.23ms
step:339/1480 train_time:48444ms step_avg:147.25ms
step:340/1480 train_time:48598ms step_avg:147.27ms
step:341/1480 train_time:48753ms step_avg:147.29ms
step:342/1480 train_time:48906ms step_avg:147.31ms
step:343/1480 train_time:49061ms step_avg:147.33ms
step:344/1480 train_time:49216ms step_avg:147.35ms
step:345/1480 train_time:49370ms step_avg:147.37ms
step:346/1480 train_time:49524ms step_avg:147.39ms
step:347/1480 train_time:49678ms step_avg:147.41ms
step:348/1480 train_time:49835ms step_avg:147.44ms
step:349/1480 train_time:49989ms step_avg:147.46ms
step:350/1480 train_time:50143ms step_avg:147.48ms
step:351/1480 train_time:50296ms step_avg:147.50ms
step:352/1480 train_time:50450ms step_avg:147.52ms
step:353/1480 train_time:50603ms step_avg:147.53ms
step:354/1480 train_time:50756ms step_avg:147.55ms
step:355/1480 train_time:50910ms step_avg:147.57ms
step:356/1480 train_time:51065ms step_avg:147.59ms
step:357/1480 train_time:51219ms step_avg:147.61ms
step:358/1480 train_time:51374ms step_avg:147.63ms
step:359/1480 train_time:51528ms step_avg:147.65ms
step:360/1480 train_time:51684ms step_avg:147.67ms
step:361/1480 train_time:51838ms step_avg:147.69ms
step:362/1480 train_time:51992ms step_avg:147.70ms
step:363/1480 train_time:52144ms step_avg:147.72ms
step:364/1480 train_time:52298ms step_avg:147.73ms
step:365/1480 train_time:52453ms step_avg:147.75ms
step:366/1480 train_time:52606ms step_avg:147.77ms
step:367/1480 train_time:52759ms step_avg:147.79ms
step:368/1480 train_time:52912ms step_avg:147.80ms
step:369/1480 train_time:53067ms step_avg:147.82ms
step:370/1480 train_time:53219ms step_avg:147.83ms
step:371/1480 train_time:53373ms step_avg:147.85ms
step:372/1480 train_time:53527ms step_avg:147.86ms
step:373/1480 train_time:53680ms step_avg:147.88ms
step:374/1480 train_time:53835ms step_avg:147.90ms
step:375/1480 train_time:53990ms step_avg:147.92ms
step:375/1480 val_loss:3.8044 train_time:54050ms step_avg:148.08ms
step:376/1480 train_time:54148ms step_avg:147.95ms
step:377/1480 train_time:54303ms step_avg:147.96ms
step:378/1480 train_time:54456ms step_avg:147.98ms
step:379/1480 train_time:54609ms step_avg:147.99ms
step:380/1480 train_time:54761ms step_avg:148.00ms
step:381/1480 train_time:54914ms step_avg:148.01ms
step:382/1480 train_time:55067ms step_avg:148.03ms
step:383/1480 train_time:55221ms step_avg:148.05ms
step:384/1480 train_time:55376ms step_avg:148.06ms
step:385/1480 train_time:55529ms step_avg:148.08ms
step:386/1480 train_time:55685ms step_avg:148.10ms
step:387/1480 train_time:55839ms step_avg:148.11ms
step:388/1480 train_time:55991ms step_avg:148.12ms
step:389/1480 train_time:56144ms step_avg:148.14ms
step:390/1480 train_time:56299ms step_avg:148.15ms
step:391/1480 train_time:56453ms step_avg:148.17ms
step:392/1480 train_time:56604ms step_avg:148.18ms
step:393/1480 train_time:56757ms step_avg:148.19ms
step:394/1480 train_time:56909ms step_avg:148.20ms
step:395/1480 train_time:57064ms step_avg:148.22ms
step:396/1480 train_time:57215ms step_avg:148.23ms
step:397/1480 train_time:57371ms step_avg:148.24ms
step:398/1480 train_time:57524ms step_avg:148.26ms
step:399/1480 train_time:57677ms step_avg:148.27ms
step:400/1480 train_time:57832ms step_avg:148.29ms
step:401/1480 train_time:57985ms step_avg:148.30ms
step:402/1480 train_time:58139ms step_avg:148.32ms
step:403/1480 train_time:58294ms step_avg:148.33ms
step:404/1480 train_time:58449ms step_avg:148.35ms
step:405/1480 train_time:58602ms step_avg:148.36ms
step:406/1480 train_time:58756ms step_avg:148.37ms
step:407/1480 train_time:58911ms step_avg:148.39ms
step:408/1480 train_time:59065ms step_avg:148.40ms
step:409/1480 train_time:59219ms step_avg:148.42ms
step:410/1480 train_time:59373ms step_avg:148.43ms
step:411/1480 train_time:59527ms step_avg:148.45ms
step:412/1480 train_time:59680ms step_avg:148.46ms
step:413/1480 train_time:59835ms step_avg:148.47ms
step:414/1480 train_time:59990ms step_avg:148.49ms
step:415/1480 train_time:60143ms step_avg:148.50ms
step:416/1480 train_time:60297ms step_avg:148.51ms
step:417/1480 train_time:60452ms step_avg:148.53ms
step:418/1480 train_time:60606ms step_avg:148.54ms
step:419/1480 train_time:60758ms step_avg:148.55ms
step:420/1480 train_time:60912ms step_avg:148.57ms
step:421/1480 train_time:61068ms step_avg:148.58ms
step:422/1480 train_time:61221ms step_avg:148.59ms
step:423/1480 train_time:61374ms step_avg:148.61ms
step:424/1480 train_time:61527ms step_avg:148.62ms
step:425/1480 train_time:61681ms step_avg:148.63ms
step:426/1480 train_time:61835ms step_avg:148.64ms
step:427/1480 train_time:61990ms step_avg:148.66ms
step:428/1480 train_time:62144ms step_avg:148.67ms
step:429/1480 train_time:62297ms step_avg:148.68ms
step:430/1480 train_time:62452ms step_avg:148.69ms
step:431/1480 train_time:62606ms step_avg:148.71ms
step:432/1480 train_time:62760ms step_avg:148.72ms
step:433/1480 train_time:62913ms step_avg:148.73ms
step:434/1480 train_time:63068ms step_avg:148.75ms
step:435/1480 train_time:63222ms step_avg:148.76ms
step:436/1480 train_time:63376ms step_avg:148.77ms
step:437/1480 train_time:63530ms step_avg:148.78ms
step:438/1480 train_time:63684ms step_avg:148.80ms
step:439/1480 train_time:63839ms step_avg:148.81ms
step:440/1480 train_time:63994ms step_avg:148.82ms
step:441/1480 train_time:64151ms step_avg:148.84ms
step:442/1480 train_time:64309ms step_avg:148.86ms
step:443/1480 train_time:64464ms step_avg:148.88ms
step:444/1480 train_time:64620ms step_avg:148.89ms
step:445/1480 train_time:64776ms step_avg:148.91ms
step:446/1480 train_time:64934ms step_avg:148.93ms
step:447/1480 train_time:65092ms step_avg:148.95ms
step:448/1480 train_time:65249ms step_avg:148.97ms
step:449/1480 train_time:65406ms step_avg:148.99ms
step:450/1480 train_time:65563ms step_avg:149.01ms
step:451/1480 train_time:65719ms step_avg:149.02ms
step:452/1480 train_time:65876ms step_avg:149.04ms
step:453/1480 train_time:66034ms step_avg:149.06ms
step:454/1480 train_time:66191ms step_avg:149.08ms
step:455/1480 train_time:66348ms step_avg:149.10ms
step:456/1480 train_time:66504ms step_avg:149.11ms
step:457/1480 train_time:66661ms step_avg:149.13ms
step:458/1480 train_time:66818ms step_avg:149.15ms
step:459/1480 train_time:66976ms step_avg:149.17ms
step:460/1480 train_time:67134ms step_avg:149.19ms
step:461/1480 train_time:67293ms step_avg:149.21ms
step:462/1480 train_time:67451ms step_avg:149.23ms
step:463/1480 train_time:67608ms step_avg:149.25ms
step:464/1480 train_time:67765ms step_avg:149.26ms
step:465/1480 train_time:67922ms step_avg:149.28ms
step:466/1480 train_time:68079ms step_avg:149.30ms
step:467/1480 train_time:68238ms step_avg:149.32ms
step:468/1480 train_time:68394ms step_avg:149.33ms
step:469/1480 train_time:68551ms step_avg:149.35ms
step:470/1480 train_time:68708ms step_avg:149.37ms
step:471/1480 train_time:68865ms step_avg:149.38ms
step:472/1480 train_time:69021ms step_avg:149.40ms
step:473/1480 train_time:69178ms step_avg:149.41ms
step:474/1480 train_time:69335ms step_avg:149.43ms
step:475/1480 train_time:69491ms step_avg:149.44ms
step:476/1480 train_time:69648ms step_avg:149.46ms
step:477/1480 train_time:69805ms step_avg:149.48ms
step:478/1480 train_time:69962ms step_avg:149.49ms
step:479/1480 train_time:70118ms step_avg:149.51ms
step:480/1480 train_time:70276ms step_avg:149.52ms
step:481/1480 train_time:70434ms step_avg:149.54ms
step:482/1480 train_time:70591ms step_avg:149.56ms
step:483/1480 train_time:70747ms step_avg:149.57ms
step:484/1480 train_time:70902ms step_avg:149.58ms
step:485/1480 train_time:71059ms step_avg:149.60ms
step:486/1480 train_time:71216ms step_avg:149.61ms
step:487/1480 train_time:71374ms step_avg:149.63ms
step:488/1480 train_time:71532ms step_avg:149.65ms
step:489/1480 train_time:71689ms step_avg:149.66ms
step:490/1480 train_time:71846ms step_avg:149.68ms
step:491/1480 train_time:72001ms step_avg:149.69ms
step:492/1480 train_time:72159ms step_avg:149.71ms
step:493/1480 train_time:72316ms step_avg:149.72ms
step:494/1480 train_time:72474ms step_avg:149.74ms
step:495/1480 train_time:72633ms step_avg:149.76ms
step:496/1480 train_time:72791ms step_avg:149.78ms
step:497/1480 train_time:72948ms step_avg:149.79ms
step:498/1480 train_time:73104ms step_avg:149.80ms
step:499/1480 train_time:73260ms step_avg:149.82ms
step:500/1480 train_time:73418ms step_avg:149.83ms
step:500/1480 val_loss:3.6861 train_time:73480ms step_avg:149.96ms
step:501/1480 train_time:73579ms step_avg:149.86ms
step:502/1480 train_time:73738ms step_avg:149.87ms
step:503/1480 train_time:73894ms step_avg:149.89ms
step:504/1480 train_time:74049ms step_avg:149.90ms
step:505/1480 train_time:74204ms step_avg:149.91ms
step:506/1480 train_time:74362ms step_avg:149.92ms
step:507/1480 train_time:74518ms step_avg:149.94ms
step:508/1480 train_time:74675ms step_avg:149.95ms
step:509/1480 train_time:74831ms step_avg:149.96ms
step:510/1480 train_time:74988ms step_avg:149.98ms
step:511/1480 train_time:75144ms step_avg:149.99ms
step:512/1480 train_time:75302ms step_avg:150.00ms
step:513/1480 train_time:75459ms step_avg:150.02ms
step:514/1480 train_time:75616ms step_avg:150.03ms
step:515/1480 train_time:75772ms step_avg:150.04ms
step:516/1480 train_time:75930ms step_avg:150.06ms
step:517/1480 train_time:76087ms step_avg:150.07ms
step:518/1480 train_time:76244ms step_avg:150.09ms
step:519/1480 train_time:76402ms step_avg:150.10ms
step:520/1480 train_time:76559ms step_avg:150.12ms
step:521/1480 train_time:76715ms step_avg:150.13ms
step:522/1480 train_time:76871ms step_avg:150.14ms
step:523/1480 train_time:77029ms step_avg:150.15ms
step:524/1480 train_time:77186ms step_avg:150.17ms
step:525/1480 train_time:77344ms step_avg:150.18ms
step:526/1480 train_time:77502ms step_avg:150.20ms
step:527/1480 train_time:77659ms step_avg:150.21ms
step:528/1480 train_time:77815ms step_avg:150.22ms
step:529/1480 train_time:77971ms step_avg:150.23ms
step:530/1480 train_time:78129ms step_avg:150.25ms
step:531/1480 train_time:78286ms step_avg:150.26ms
step:532/1480 train_time:78443ms step_avg:150.27ms
step:533/1480 train_time:78600ms step_avg:150.29ms
step:534/1480 train_time:78755ms step_avg:150.30ms
step:535/1480 train_time:78912ms step_avg:150.31ms
step:536/1480 train_time:79069ms step_avg:150.32ms
step:537/1480 train_time:79226ms step_avg:150.33ms
step:538/1480 train_time:79383ms step_avg:150.35ms
step:539/1480 train_time:79543ms step_avg:150.37ms
step:540/1480 train_time:79703ms step_avg:150.38ms
step:541/1480 train_time:79861ms step_avg:150.40ms
step:542/1480 train_time:80018ms step_avg:150.41ms
step:543/1480 train_time:80173ms step_avg:150.42ms
step:544/1480 train_time:80329ms step_avg:150.43ms
step:545/1480 train_time:80486ms step_avg:150.44ms
step:546/1480 train_time:80645ms step_avg:150.46ms
step:547/1480 train_time:80802ms step_avg:150.47ms
step:548/1480 train_time:80961ms step_avg:150.49ms
step:549/1480 train_time:81118ms step_avg:150.50ms
step:550/1480 train_time:81274ms step_avg:150.51ms
step:551/1480 train_time:81431ms step_avg:150.52ms
step:552/1480 train_time:81590ms step_avg:150.53ms
step:553/1480 train_time:81750ms step_avg:150.55ms
step:554/1480 train_time:81910ms step_avg:150.57ms
step:555/1480 train_time:82071ms step_avg:150.59ms
step:556/1480 train_time:82230ms step_avg:150.61ms
step:557/1480 train_time:82389ms step_avg:150.62ms
step:558/1480 train_time:82548ms step_avg:150.63ms
step:559/1480 train_time:82705ms step_avg:150.65ms
step:560/1480 train_time:82866ms step_avg:150.66ms
step:561/1480 train_time:83025ms step_avg:150.68ms
step:562/1480 train_time:83185ms step_avg:150.70ms
step:563/1480 train_time:83345ms step_avg:150.71ms
step:564/1480 train_time:83505ms step_avg:150.73ms
step:565/1480 train_time:83665ms step_avg:150.75ms
step:566/1480 train_time:83825ms step_avg:150.76ms
step:567/1480 train_time:83985ms step_avg:150.78ms
step:568/1480 train_time:84144ms step_avg:150.80ms
step:569/1480 train_time:84304ms step_avg:150.81ms
step:570/1480 train_time:84464ms step_avg:150.83ms
step:571/1480 train_time:84625ms step_avg:150.85ms
step:572/1480 train_time:84785ms step_avg:150.86ms
step:573/1480 train_time:84944ms step_avg:150.88ms
step:574/1480 train_time:85106ms step_avg:150.90ms
step:575/1480 train_time:85267ms step_avg:150.91ms
step:576/1480 train_time:85425ms step_avg:150.93ms
step:577/1480 train_time:85585ms step_avg:150.94ms
step:578/1480 train_time:85744ms step_avg:150.96ms
step:579/1480 train_time:85904ms step_avg:150.97ms
step:580/1480 train_time:86063ms step_avg:150.99ms
step:581/1480 train_time:86226ms step_avg:151.01ms
step:582/1480 train_time:86386ms step_avg:151.02ms
step:583/1480 train_time:86545ms step_avg:151.04ms
step:584/1480 train_time:86705ms step_avg:151.05ms
step:585/1480 train_time:86864ms step_avg:151.07ms
step:586/1480 train_time:87024ms step_avg:151.08ms
step:587/1480 train_time:87184ms step_avg:151.10ms
step:588/1480 train_time:87343ms step_avg:151.11ms
step:589/1480 train_time:87503ms step_avg:151.13ms
step:590/1480 train_time:87665ms step_avg:151.15ms
step:591/1480 train_time:87825ms step_avg:151.16ms
step:592/1480 train_time:87984ms step_avg:151.18ms
step:593/1480 train_time:88145ms step_avg:151.19ms
step:594/1480 train_time:88306ms step_avg:151.21ms
step:595/1480 train_time:88468ms step_avg:151.23ms
step:596/1480 train_time:88629ms step_avg:151.24ms
step:597/1480 train_time:88787ms step_avg:151.26ms
step:598/1480 train_time:88946ms step_avg:151.27ms
step:599/1480 train_time:89105ms step_avg:151.28ms
step:600/1480 train_time:89265ms step_avg:151.30ms
step:601/1480 train_time:89424ms step_avg:151.31ms
step:602/1480 train_time:89584ms step_avg:151.32ms
step:603/1480 train_time:89746ms step_avg:151.34ms
step:604/1480 train_time:89906ms step_avg:151.36ms
step:605/1480 train_time:90066ms step_avg:151.37ms
step:606/1480 train_time:90227ms step_avg:151.39ms
step:607/1480 train_time:90389ms step_avg:151.41ms
step:608/1480 train_time:90548ms step_avg:151.42ms
step:609/1480 train_time:90707ms step_avg:151.43ms
step:610/1480 train_time:90866ms step_avg:151.44ms
step:611/1480 train_time:91026ms step_avg:151.46ms
step:612/1480 train_time:91185ms step_avg:151.47ms
step:613/1480 train_time:91346ms step_avg:151.49ms
step:614/1480 train_time:91506ms step_avg:151.50ms
step:615/1480 train_time:91665ms step_avg:151.51ms
step:616/1480 train_time:91824ms step_avg:151.52ms
step:617/1480 train_time:91983ms step_avg:151.54ms
step:618/1480 train_time:92143ms step_avg:151.55ms
step:619/1480 train_time:92302ms step_avg:151.56ms
step:620/1480 train_time:92462ms step_avg:151.58ms
step:621/1480 train_time:92622ms step_avg:151.59ms
step:622/1480 train_time:92783ms step_avg:151.61ms
step:623/1480 train_time:92945ms step_avg:151.62ms
step:624/1480 train_time:93105ms step_avg:151.64ms
step:625/1480 train_time:93265ms step_avg:151.65ms
step:625/1480 val_loss:3.6060 train_time:93328ms step_avg:151.75ms
step:626/1480 train_time:93428ms step_avg:151.67ms
step:627/1480 train_time:93587ms step_avg:151.68ms
step:628/1480 train_time:93745ms step_avg:151.69ms
step:629/1480 train_time:93903ms step_avg:151.70ms
step:630/1480 train_time:94060ms step_avg:151.71ms
step:631/1480 train_time:94218ms step_avg:151.72ms
step:632/1480 train_time:94376ms step_avg:151.73ms
step:633/1480 train_time:94537ms step_avg:151.75ms
step:634/1480 train_time:94698ms step_avg:151.76ms
step:635/1480 train_time:94857ms step_avg:151.77ms
step:636/1480 train_time:95017ms step_avg:151.78ms
step:637/1480 train_time:95175ms step_avg:151.80ms
step:638/1480 train_time:95334ms step_avg:151.81ms
step:639/1480 train_time:95494ms step_avg:151.82ms
step:640/1480 train_time:95654ms step_avg:151.83ms
step:641/1480 train_time:95815ms step_avg:151.85ms
step:642/1480 train_time:95974ms step_avg:151.86ms
step:643/1480 train_time:96135ms step_avg:151.87ms
step:644/1480 train_time:96294ms step_avg:151.88ms
step:645/1480 train_time:96452ms step_avg:151.89ms
step:646/1480 train_time:96612ms step_avg:151.91ms
step:647/1480 train_time:96773ms step_avg:151.92ms
step:648/1480 train_time:96936ms step_avg:151.94ms
step:649/1480 train_time:97095ms step_avg:151.95ms
step:650/1480 train_time:97255ms step_avg:151.96ms
step:651/1480 train_time:97415ms step_avg:151.97ms
step:652/1480 train_time:97574ms step_avg:151.98ms
step:653/1480 train_time:97734ms step_avg:152.00ms
step:654/1480 train_time:97894ms step_avg:152.01ms
step:655/1480 train_time:98054ms step_avg:152.02ms
step:656/1480 train_time:98214ms step_avg:152.03ms
step:657/1480 train_time:98374ms step_avg:152.05ms
step:658/1480 train_time:98535ms step_avg:152.06ms
step:659/1480 train_time:98696ms step_avg:152.07ms
step:660/1480 train_time:98859ms step_avg:152.09ms
step:661/1480 train_time:99021ms step_avg:152.11ms
step:662/1480 train_time:99181ms step_avg:152.12ms
step:663/1480 train_time:99340ms step_avg:152.13ms
step:664/1480 train_time:99501ms step_avg:152.14ms
step:665/1480 train_time:99663ms step_avg:152.16ms
step:666/1480 train_time:99823ms step_avg:152.17ms
step:667/1480 train_time:99985ms step_avg:152.18ms
step:668/1480 train_time:100147ms step_avg:152.20ms
step:669/1480 train_time:100310ms step_avg:152.21ms
step:670/1480 train_time:100470ms step_avg:152.23ms
step:671/1480 train_time:100631ms step_avg:152.24ms
step:672/1480 train_time:100794ms step_avg:152.26ms
step:673/1480 train_time:100957ms step_avg:152.27ms
step:674/1480 train_time:101118ms step_avg:152.29ms
step:675/1480 train_time:101279ms step_avg:152.30ms
step:676/1480 train_time:101440ms step_avg:152.31ms
step:677/1480 train_time:101601ms step_avg:152.33ms
step:678/1480 train_time:101760ms step_avg:152.34ms
step:679/1480 train_time:101922ms step_avg:152.35ms
step:680/1480 train_time:102086ms step_avg:152.37ms
step:681/1480 train_time:102248ms step_avg:152.38ms
step:682/1480 train_time:102412ms step_avg:152.40ms
step:683/1480 train_time:102574ms step_avg:152.41ms
step:684/1480 train_time:102735ms step_avg:152.43ms
step:685/1480 train_time:102899ms step_avg:152.44ms
step:686/1480 train_time:103058ms step_avg:152.45ms
step:687/1480 train_time:103219ms step_avg:152.47ms
step:688/1480 train_time:103381ms step_avg:152.48ms
step:689/1480 train_time:103543ms step_avg:152.49ms
step:690/1480 train_time:103707ms step_avg:152.51ms
step:691/1480 train_time:103869ms step_avg:152.52ms
step:692/1480 train_time:104031ms step_avg:152.54ms
step:693/1480 train_time:104194ms step_avg:152.55ms
step:694/1480 train_time:104356ms step_avg:152.57ms
step:695/1480 train_time:104518ms step_avg:152.58ms
step:696/1480 train_time:104677ms step_avg:152.59ms
step:697/1480 train_time:104840ms step_avg:152.61ms
step:698/1480 train_time:105001ms step_avg:152.62ms
step:699/1480 train_time:105163ms step_avg:152.63ms
step:700/1480 train_time:105324ms step_avg:152.64ms
step:701/1480 train_time:105483ms step_avg:152.65ms
step:702/1480 train_time:105642ms step_avg:152.66ms
step:703/1480 train_time:105803ms step_avg:152.67ms
step:704/1480 train_time:105962ms step_avg:152.68ms
step:705/1480 train_time:106126ms step_avg:152.70ms
step:706/1480 train_time:106290ms step_avg:152.72ms
step:707/1480 train_time:106452ms step_avg:152.73ms
step:708/1480 train_time:106614ms step_avg:152.74ms
step:709/1480 train_time:106775ms step_avg:152.75ms
step:710/1480 train_time:106937ms step_avg:152.77ms
step:711/1480 train_time:107100ms step_avg:152.78ms
step:712/1480 train_time:107262ms step_avg:152.80ms
step:713/1480 train_time:107426ms step_avg:152.81ms
step:714/1480 train_time:107586ms step_avg:152.82ms
step:715/1480 train_time:107746ms step_avg:152.83ms
step:716/1480 train_time:107908ms step_avg:152.84ms
step:717/1480 train_time:108070ms step_avg:152.86ms
step:718/1480 train_time:108230ms step_avg:152.87ms
step:719/1480 train_time:108391ms step_avg:152.88ms
step:720/1480 train_time:108555ms step_avg:152.89ms
step:721/1480 train_time:108717ms step_avg:152.91ms
step:722/1480 train_time:108877ms step_avg:152.92ms
step:723/1480 train_time:109038ms step_avg:152.93ms
step:724/1480 train_time:109201ms step_avg:152.94ms
step:725/1480 train_time:109362ms step_avg:152.95ms
step:726/1480 train_time:109526ms step_avg:152.97ms
step:727/1480 train_time:109690ms step_avg:152.98ms
step:728/1480 train_time:109850ms step_avg:153.00ms
step:729/1480 train_time:110013ms step_avg:153.01ms
step:730/1480 train_time:110176ms step_avg:153.02ms
step:731/1480 train_time:110338ms step_avg:153.03ms
step:732/1480 train_time:110498ms step_avg:153.04ms
step:733/1480 train_time:110659ms step_avg:153.06ms
step:734/1480 train_time:110820ms step_avg:153.07ms
step:735/1480 train_time:110980ms step_avg:153.08ms
step:736/1480 train_time:111142ms step_avg:153.09ms
step:737/1480 train_time:111303ms step_avg:153.10ms
step:738/1480 train_time:111464ms step_avg:153.11ms
step:739/1480 train_time:111624ms step_avg:153.12ms
step:740/1480 train_time:111788ms step_avg:153.13ms
step:741/1480 train_time:111952ms step_avg:153.15ms
step:742/1480 train_time:112115ms step_avg:153.16ms
step:743/1480 train_time:112277ms step_avg:153.17ms
step:744/1480 train_time:112440ms step_avg:153.19ms
step:745/1480 train_time:112603ms step_avg:153.20ms
step:746/1480 train_time:112761ms step_avg:153.21ms
step:747/1480 train_time:112923ms step_avg:153.22ms
step:748/1480 train_time:113090ms step_avg:153.24ms
step:749/1480 train_time:113255ms step_avg:153.25ms
step:750/1480 train_time:113416ms step_avg:153.26ms
step:750/1480 val_loss:3.5475 train_time:113480ms step_avg:153.35ms
step:751/1480 train_time:113581ms step_avg:153.28ms
step:752/1480 train_time:113743ms step_avg:153.29ms
step:753/1480 train_time:113903ms step_avg:153.30ms
step:754/1480 train_time:114064ms step_avg:153.31ms
step:755/1480 train_time:114225ms step_avg:153.32ms
step:756/1480 train_time:114386ms step_avg:153.33ms
step:757/1480 train_time:114552ms step_avg:153.35ms
step:758/1480 train_time:114712ms step_avg:153.36ms
step:759/1480 train_time:114875ms step_avg:153.37ms
step:760/1480 train_time:115039ms step_avg:153.38ms
step:761/1480 train_time:115202ms step_avg:153.40ms
step:762/1480 train_time:115363ms step_avg:153.41ms
step:763/1480 train_time:115525ms step_avg:153.42ms
step:764/1480 train_time:115686ms step_avg:153.43ms
step:765/1480 train_time:115847ms step_avg:153.44ms
step:766/1480 train_time:116008ms step_avg:153.45ms
step:767/1480 train_time:116170ms step_avg:153.46ms
step:768/1480 train_time:116331ms step_avg:153.47ms
step:769/1480 train_time:116496ms step_avg:153.49ms
step:770/1480 train_time:116660ms step_avg:153.50ms
step:771/1480 train_time:116823ms step_avg:153.51ms
step:772/1480 train_time:116985ms step_avg:153.52ms
step:773/1480 train_time:117147ms step_avg:153.54ms
step:774/1480 train_time:117308ms step_avg:153.54ms
step:775/1480 train_time:117469ms step_avg:153.55ms
step:776/1480 train_time:117635ms step_avg:153.57ms
step:777/1480 train_time:117802ms step_avg:153.59ms
step:778/1480 train_time:117965ms step_avg:153.60ms
step:779/1480 train_time:118126ms step_avg:153.61ms
step:780/1480 train_time:118289ms step_avg:153.62ms
step:781/1480 train_time:118452ms step_avg:153.63ms
step:782/1480 train_time:118616ms step_avg:153.65ms
step:783/1480 train_time:118778ms step_avg:153.66ms
step:784/1480 train_time:118942ms step_avg:153.67ms
step:785/1480 train_time:119105ms step_avg:153.68ms
step:786/1480 train_time:119269ms step_avg:153.70ms
step:787/1480 train_time:119431ms step_avg:153.71ms
step:788/1480 train_time:119597ms step_avg:153.72ms
step:789/1480 train_time:119760ms step_avg:153.74ms
step:790/1480 train_time:119925ms step_avg:153.75ms
step:791/1480 train_time:120090ms step_avg:153.77ms
step:792/1480 train_time:120255ms step_avg:153.78ms
step:793/1480 train_time:120418ms step_avg:153.79ms
step:794/1480 train_time:120583ms step_avg:153.80ms
step:795/1480 train_time:120747ms step_avg:153.82ms
step:796/1480 train_time:120912ms step_avg:153.83ms
step:797/1480 train_time:121079ms step_avg:153.85ms
step:798/1480 train_time:121242ms step_avg:153.86ms
step:799/1480 train_time:121408ms step_avg:153.88ms
step:800/1480 train_time:121570ms step_avg:153.89ms
step:801/1480 train_time:121731ms step_avg:153.90ms
step:802/1480 train_time:121902ms step_avg:153.92ms
step:803/1480 train_time:122065ms step_avg:153.93ms
step:804/1480 train_time:122226ms step_avg:153.94ms
step:805/1480 train_time:122391ms step_avg:153.95ms
step:806/1480 train_time:122553ms step_avg:153.96ms
step:807/1480 train_time:122714ms step_avg:153.97ms
step:808/1480 train_time:122880ms step_avg:153.99ms
step:809/1480 train_time:123042ms step_avg:153.99ms
step:810/1480 train_time:123204ms step_avg:154.01ms
step:811/1480 train_time:123367ms step_avg:154.02ms
step:812/1480 train_time:123529ms step_avg:154.03ms
step:813/1480 train_time:123688ms step_avg:154.03ms
step:814/1480 train_time:123851ms step_avg:154.04ms
step:815/1480 train_time:124014ms step_avg:154.05ms
step:816/1480 train_time:124181ms step_avg:154.07ms
step:817/1480 train_time:124344ms step_avg:154.08ms
step:818/1480 train_time:124505ms step_avg:154.09ms
step:819/1480 train_time:124668ms step_avg:154.10ms
step:820/1480 train_time:124832ms step_avg:154.11ms
step:821/1480 train_time:124994ms step_avg:154.12ms
step:822/1480 train_time:125160ms step_avg:154.14ms
step:823/1480 train_time:125322ms step_avg:154.15ms
step:824/1480 train_time:125485ms step_avg:154.16ms
step:825/1480 train_time:125647ms step_avg:154.17ms
step:826/1480 train_time:125813ms step_avg:154.18ms
step:827/1480 train_time:125979ms step_avg:154.20ms
step:828/1480 train_time:126142ms step_avg:154.21ms
step:829/1480 train_time:126306ms step_avg:154.22ms
step:830/1480 train_time:126470ms step_avg:154.23ms
step:831/1480 train_time:126633ms step_avg:154.24ms
step:832/1480 train_time:126799ms step_avg:154.26ms
step:833/1480 train_time:126964ms step_avg:154.27ms
step:834/1480 train_time:127128ms step_avg:154.28ms
step:835/1480 train_time:127291ms step_avg:154.29ms
step:836/1480 train_time:127456ms step_avg:154.31ms
step:837/1480 train_time:127619ms step_avg:154.32ms
step:838/1480 train_time:127783ms step_avg:154.33ms
step:839/1480 train_time:127945ms step_avg:154.34ms
step:840/1480 train_time:128106ms step_avg:154.34ms
step:841/1480 train_time:128267ms step_avg:154.35ms
step:842/1480 train_time:128429ms step_avg:154.36ms
step:843/1480 train_time:128590ms step_avg:154.37ms
step:844/1480 train_time:128752ms step_avg:154.38ms
step:845/1480 train_time:128917ms step_avg:154.39ms
step:846/1480 train_time:129081ms step_avg:154.40ms
step:847/1480 train_time:129246ms step_avg:154.42ms
step:848/1480 train_time:129408ms step_avg:154.42ms
step:849/1480 train_time:129569ms step_avg:154.43ms
step:850/1480 train_time:129733ms step_avg:154.44ms
step:851/1480 train_time:129900ms step_avg:154.46ms
step:852/1480 train_time:130063ms step_avg:154.47ms
step:853/1480 train_time:130225ms step_avg:154.48ms
step:854/1480 train_time:130390ms step_avg:154.49ms
step:855/1480 train_time:130551ms step_avg:154.50ms
step:856/1480 train_time:130714ms step_avg:154.51ms
step:857/1480 train_time:130881ms step_avg:154.52ms
step:858/1480 train_time:131045ms step_avg:154.53ms
step:859/1480 train_time:131208ms step_avg:154.54ms
step:860/1480 train_time:131369ms step_avg:154.55ms
step:861/1480 train_time:131534ms step_avg:154.56ms
step:862/1480 train_time:131704ms step_avg:154.58ms
step:863/1480 train_time:131871ms step_avg:154.60ms
step:864/1480 train_time:132034ms step_avg:154.61ms
step:865/1480 train_time:132195ms step_avg:154.61ms
step:866/1480 train_time:132363ms step_avg:154.63ms
step:867/1480 train_time:132526ms step_avg:154.64ms
step:868/1480 train_time:132687ms step_avg:154.65ms
step:869/1480 train_time:132849ms step_avg:154.66ms
step:870/1480 train_time:133013ms step_avg:154.67ms
step:871/1480 train_time:133176ms step_avg:154.68ms
step:872/1480 train_time:133341ms step_avg:154.69ms
step:873/1480 train_time:133503ms step_avg:154.70ms
step:874/1480 train_time:133668ms step_avg:154.71ms
step:875/1480 train_time:133833ms step_avg:154.72ms
step:875/1480 val_loss:3.5055 train_time:133898ms step_avg:154.80ms
step:876/1480 train_time:133998ms step_avg:154.73ms
step:877/1480 train_time:134163ms step_avg:154.74ms
step:878/1480 train_time:134326ms step_avg:154.75ms
step:879/1480 train_time:134490ms step_avg:154.76ms
step:880/1480 train_time:134653ms step_avg:154.77ms
step:881/1480 train_time:134815ms step_avg:154.78ms
step:882/1480 train_time:134979ms step_avg:154.79ms
step:883/1480 train_time:135145ms step_avg:154.81ms
step:884/1480 train_time:135312ms step_avg:154.82ms
step:885/1480 train_time:135477ms step_avg:154.83ms
step:886/1480 train_time:135643ms step_avg:154.84ms
step:887/1480 train_time:135812ms step_avg:154.86ms
step:888/1480 train_time:135984ms step_avg:154.88ms
step:889/1480 train_time:136151ms step_avg:154.89ms
step:890/1480 train_time:136314ms step_avg:154.90ms
step:891/1480 train_time:136479ms step_avg:154.91ms
step:892/1480 train_time:136644ms step_avg:154.93ms
step:893/1480 train_time:136808ms step_avg:154.94ms
step:894/1480 train_time:136975ms step_avg:154.95ms
step:895/1480 train_time:137141ms step_avg:154.96ms
step:896/1480 train_time:137307ms step_avg:154.97ms
step:897/1480 train_time:137473ms step_avg:154.99ms
step:898/1480 train_time:137642ms step_avg:155.00ms
step:899/1480 train_time:137806ms step_avg:155.01ms
step:900/1480 train_time:137971ms step_avg:155.02ms
step:901/1480 train_time:138136ms step_avg:155.03ms
step:902/1480 train_time:138298ms step_avg:155.04ms
step:903/1480 train_time:138471ms step_avg:155.06ms
step:904/1480 train_time:138636ms step_avg:155.07ms
step:905/1480 train_time:138798ms step_avg:155.08ms
step:906/1480 train_time:138965ms step_avg:155.09ms
step:907/1480 train_time:139133ms step_avg:155.11ms
step:908/1480 train_time:139294ms step_avg:155.12ms
step:909/1480 train_time:139458ms step_avg:155.13ms
step:910/1480 train_time:139630ms step_avg:155.14ms
step:911/1480 train_time:139794ms step_avg:155.15ms
step:912/1480 train_time:139960ms step_avg:155.17ms
step:913/1480 train_time:140131ms step_avg:155.18ms
step:914/1480 train_time:140298ms step_avg:155.20ms
step:915/1480 train_time:140469ms step_avg:155.21ms
step:916/1480 train_time:140632ms step_avg:155.22ms
step:917/1480 train_time:140795ms step_avg:155.23ms
step:918/1480 train_time:140961ms step_avg:155.24ms
step:919/1480 train_time:141132ms step_avg:155.26ms
step:920/1480 train_time:141297ms step_avg:155.27ms
step:921/1480 train_time:141462ms step_avg:155.28ms
step:922/1480 train_time:141630ms step_avg:155.30ms
step:923/1480 train_time:141792ms step_avg:155.30ms
step:924/1480 train_time:141956ms step_avg:155.31ms
step:925/1480 train_time:142122ms step_avg:155.32ms
step:926/1480 train_time:142286ms step_avg:155.33ms
step:927/1480 train_time:142450ms step_avg:155.34ms
step:928/1480 train_time:142617ms step_avg:155.36ms
step:929/1480 train_time:142782ms step_avg:155.37ms
step:930/1480 train_time:142949ms step_avg:155.38ms
step:931/1480 train_time:143112ms step_avg:155.39ms
step:932/1480 train_time:143277ms step_avg:155.40ms
step:933/1480 train_time:143445ms step_avg:155.41ms
step:934/1480 train_time:143612ms step_avg:155.42ms
step:935/1480 train_time:143782ms step_avg:155.44ms
step:936/1480 train_time:143951ms step_avg:155.45ms
step:937/1480 train_time:144121ms step_avg:155.47ms
step:938/1480 train_time:144284ms step_avg:155.48ms
step:939/1480 train_time:144454ms step_avg:155.49ms
step:940/1480 train_time:144621ms step_avg:155.51ms
step:941/1480 train_time:144785ms step_avg:155.52ms
step:942/1480 train_time:144951ms step_avg:155.53ms
step:943/1480 train_time:145121ms step_avg:155.54ms
step:944/1480 train_time:145292ms step_avg:155.56ms
step:945/1480 train_time:145455ms step_avg:155.57ms
step:946/1480 train_time:145624ms step_avg:155.58ms
step:947/1480 train_time:145791ms step_avg:155.59ms
step:948/1480 train_time:145956ms step_avg:155.60ms
step:949/1480 train_time:146121ms step_avg:155.61ms
step:950/1480 train_time:146285ms step_avg:155.62ms
step:951/1480 train_time:146454ms step_avg:155.64ms
step:952/1480 train_time:146618ms step_avg:155.65ms
step:953/1480 train_time:146786ms step_avg:155.66ms
step:954/1480 train_time:146955ms step_avg:155.67ms
step:955/1480 train_time:147119ms step_avg:155.68ms
step:956/1480 train_time:147284ms step_avg:155.69ms
step:957/1480 train_time:147452ms step_avg:155.70ms
step:958/1480 train_time:147620ms step_avg:155.72ms
step:959/1480 train_time:147785ms step_avg:155.73ms
step:960/1480 train_time:147953ms step_avg:155.74ms
step:961/1480 train_time:148117ms step_avg:155.75ms
step:962/1480 train_time:148280ms step_avg:155.76ms
step:963/1480 train_time:148447ms step_avg:155.77ms
step:964/1480 train_time:148616ms step_avg:155.78ms
step:965/1480 train_time:148779ms step_avg:155.79ms
step:966/1480 train_time:148945ms step_avg:155.80ms
step:967/1480 train_time:149109ms step_avg:155.81ms
step:968/1480 train_time:149273ms step_avg:155.82ms
step:969/1480 train_time:149438ms step_avg:155.83ms
step:970/1480 train_time:149600ms step_avg:155.83ms
step:971/1480 train_time:149766ms step_avg:155.84ms
step:972/1480 train_time:149932ms step_avg:155.85ms
step:973/1480 train_time:150096ms step_avg:155.86ms
step:974/1480 train_time:150265ms step_avg:155.88ms
step:975/1480 train_time:150430ms step_avg:155.89ms
step:976/1480 train_time:150595ms step_avg:155.90ms
step:977/1480 train_time:150758ms step_avg:155.90ms
step:978/1480 train_time:150925ms step_avg:155.91ms
step:979/1480 train_time:151091ms step_avg:155.93ms
step:980/1480 train_time:151256ms step_avg:155.93ms
step:981/1480 train_time:151426ms step_avg:155.95ms
step:982/1480 train_time:151588ms step_avg:155.95ms
step:983/1480 train_time:151754ms step_avg:155.96ms
step:984/1480 train_time:151918ms step_avg:155.97ms
step:985/1480 train_time:152087ms step_avg:155.99ms
step:986/1480 train_time:152254ms step_avg:156.00ms
step:987/1480 train_time:152417ms step_avg:156.00ms
step:988/1480 train_time:152584ms step_avg:156.02ms
step:989/1480 train_time:152751ms step_avg:156.03ms
step:990/1480 train_time:152919ms step_avg:156.04ms
step:991/1480 train_time:153086ms step_avg:156.05ms
step:992/1480 train_time:153260ms step_avg:156.07ms
step:993/1480 train_time:153437ms step_avg:156.09ms
step:994/1480 train_time:153602ms step_avg:156.10ms
step:995/1480 train_time:153767ms step_avg:156.11ms
step:996/1480 train_time:153931ms step_avg:156.12ms
step:997/1480 train_time:154094ms step_avg:156.12ms
step:998/1480 train_time:154257ms step_avg:156.13ms
step:999/1480 train_time:154424ms step_avg:156.14ms
step:1000/1480 train_time:154594ms step_avg:156.16ms
step:1000/1480 val_loss:3.4427 train_time:154662ms step_avg:156.22ms
step:1001/1480 train_time:154764ms step_avg:156.17ms
step:1002/1480 train_time:154930ms step_avg:156.18ms
step:1003/1480 train_time:155103ms step_avg:156.20ms
step:1004/1480 train_time:155272ms step_avg:156.21ms
step:1005/1480 train_time:155441ms step_avg:156.22ms
step:1006/1480 train_time:155607ms step_avg:156.23ms
step:1007/1480 train_time:155774ms step_avg:156.24ms
step:1008/1480 train_time:155942ms step_avg:156.25ms
step:1009/1480 train_time:156115ms step_avg:156.27ms
step:1010/1480 train_time:156281ms step_avg:156.28ms
step:1011/1480 train_time:156446ms step_avg:156.29ms
step:1012/1480 train_time:156609ms step_avg:156.30ms
step:1013/1480 train_time:156780ms step_avg:156.31ms
step:1014/1480 train_time:156945ms step_avg:156.32ms
step:1015/1480 train_time:157116ms step_avg:156.33ms
step:1016/1480 train_time:157284ms step_avg:156.35ms
step:1017/1480 train_time:157455ms step_avg:156.36ms
step:1018/1480 train_time:157624ms step_avg:156.37ms
step:1019/1480 train_time:157794ms step_avg:156.39ms
step:1020/1480 train_time:157964ms step_avg:156.40ms
step:1021/1480 train_time:158129ms step_avg:156.41ms
step:1022/1480 train_time:158296ms step_avg:156.42ms
step:1023/1480 train_time:158463ms step_avg:156.43ms
step:1024/1480 train_time:158629ms step_avg:156.44ms
step:1025/1480 train_time:158799ms step_avg:156.45ms
step:1026/1480 train_time:158965ms step_avg:156.46ms
step:1027/1480 train_time:159133ms step_avg:156.47ms
step:1028/1480 train_time:159307ms step_avg:156.49ms
step:1029/1480 train_time:159481ms step_avg:156.51ms
step:1030/1480 train_time:159647ms step_avg:156.52ms
step:1031/1480 train_time:159810ms step_avg:156.52ms
step:1032/1480 train_time:159984ms step_avg:156.54ms
step:1033/1480 train_time:160149ms step_avg:156.55ms
step:1034/1480 train_time:160318ms step_avg:156.56ms
step:1035/1480 train_time:160487ms step_avg:156.57ms
step:1036/1480 train_time:160652ms step_avg:156.58ms
step:1037/1480 train_time:160820ms step_avg:156.59ms
step:1038/1480 train_time:160986ms step_avg:156.60ms
step:1039/1480 train_time:161156ms step_avg:156.61ms
step:1040/1480 train_time:161322ms step_avg:156.62ms
step:1041/1480 train_time:161489ms step_avg:156.63ms
step:1042/1480 train_time:161652ms step_avg:156.64ms
step:1043/1480 train_time:161819ms step_avg:156.65ms
step:1044/1480 train_time:161986ms step_avg:156.66ms
step:1045/1480 train_time:162156ms step_avg:156.67ms
step:1046/1480 train_time:162324ms step_avg:156.68ms
step:1047/1480 train_time:162489ms step_avg:156.69ms
step:1048/1480 train_time:162656ms step_avg:156.70ms
step:1049/1480 train_time:162821ms step_avg:156.71ms
step:1050/1480 train_time:162990ms step_avg:156.72ms
step:1051/1480 train_time:163161ms step_avg:156.73ms
step:1052/1480 train_time:163328ms step_avg:156.74ms
step:1053/1480 train_time:163493ms step_avg:156.75ms
step:1054/1480 train_time:163663ms step_avg:156.77ms
step:1055/1480 train_time:163829ms step_avg:156.77ms
step:1056/1480 train_time:163993ms step_avg:156.78ms
step:1057/1480 train_time:164162ms step_avg:156.79ms
step:1058/1480 train_time:164329ms step_avg:156.80ms
step:1059/1480 train_time:164503ms step_avg:156.82ms
step:1060/1480 train_time:164671ms step_avg:156.83ms
step:1061/1480 train_time:164834ms step_avg:156.84ms
step:1062/1480 train_time:165001ms step_avg:156.85ms
step:1063/1480 train_time:165166ms step_avg:156.85ms
step:1064/1480 train_time:165330ms step_avg:156.86ms
step:1065/1480 train_time:165498ms step_avg:156.87ms
step:1066/1480 train_time:165666ms step_avg:156.88ms
step:1067/1480 train_time:165837ms step_avg:156.89ms
step:1068/1480 train_time:166004ms step_avg:156.90ms
step:1069/1480 train_time:166176ms step_avg:156.92ms
step:1070/1480 train_time:166342ms step_avg:156.93ms
step:1071/1480 train_time:166514ms step_avg:156.94ms
step:1072/1480 train_time:166681ms step_avg:156.95ms
step:1073/1480 train_time:166844ms step_avg:156.96ms
step:1074/1480 train_time:167011ms step_avg:156.97ms
step:1075/1480 train_time:167182ms step_avg:156.98ms
step:1076/1480 train_time:167349ms step_avg:156.99ms
step:1077/1480 train_time:167516ms step_avg:157.00ms
step:1078/1480 train_time:167691ms step_avg:157.01ms
step:1079/1480 train_time:167863ms step_avg:157.03ms
step:1080/1480 train_time:168031ms step_avg:157.04ms
step:1081/1480 train_time:168199ms step_avg:157.05ms
step:1082/1480 train_time:168366ms step_avg:157.06ms
step:1083/1480 train_time:168532ms step_avg:157.07ms
step:1084/1480 train_time:168698ms step_avg:157.07ms
step:1085/1480 train_time:168867ms step_avg:157.09ms
step:1086/1480 train_time:169035ms step_avg:157.10ms
step:1087/1480 train_time:169203ms step_avg:157.11ms
step:1088/1480 train_time:169374ms step_avg:157.12ms
step:1089/1480 train_time:169545ms step_avg:157.13ms
step:1090/1480 train_time:169717ms step_avg:157.15ms
step:1091/1480 train_time:169886ms step_avg:157.16ms
step:1092/1480 train_time:170053ms step_avg:157.17ms
step:1093/1480 train_time:170221ms step_avg:157.18ms
step:1094/1480 train_time:170387ms step_avg:157.18ms
step:1095/1480 train_time:170551ms step_avg:157.19ms
step:1096/1480 train_time:170721ms step_avg:157.20ms
step:1097/1480 train_time:170889ms step_avg:157.21ms
step:1098/1480 train_time:171063ms step_avg:157.23ms
step:1099/1480 train_time:171233ms step_avg:157.24ms
step:1100/1480 train_time:171405ms step_avg:157.25ms
step:1101/1480 train_time:171576ms step_avg:157.26ms
step:1102/1480 train_time:171747ms step_avg:157.28ms
step:1103/1480 train_time:171923ms step_avg:157.29ms
step:1104/1480 train_time:172090ms step_avg:157.30ms
step:1105/1480 train_time:172262ms step_avg:157.32ms
step:1106/1480 train_time:172430ms step_avg:157.33ms
step:1107/1480 train_time:172599ms step_avg:157.34ms
step:1108/1480 train_time:172764ms step_avg:157.34ms
step:1109/1480 train_time:172930ms step_avg:157.35ms
step:1110/1480 train_time:173097ms step_avg:157.36ms
step:1111/1480 train_time:173263ms step_avg:157.37ms
step:1112/1480 train_time:173432ms step_avg:157.38ms
step:1113/1480 train_time:173612ms step_avg:157.40ms
step:1114/1480 train_time:173785ms step_avg:157.41ms
step:1115/1480 train_time:173958ms step_avg:157.43ms
step:1116/1480 train_time:174126ms step_avg:157.44ms
step:1117/1480 train_time:174298ms step_avg:157.45ms
step:1118/1480 train_time:174472ms step_avg:157.47ms
step:1119/1480 train_time:174638ms step_avg:157.47ms
step:1120/1480 train_time:174806ms step_avg:157.48ms
step:1121/1480 train_time:174976ms step_avg:157.49ms
step:1122/1480 train_time:175144ms step_avg:157.50ms
step:1123/1480 train_time:175310ms step_avg:157.51ms
step:1124/1480 train_time:175480ms step_avg:157.52ms
step:1125/1480 train_time:175647ms step_avg:157.53ms
step:1125/1480 val_loss:3.3864 train_time:175715ms step_avg:157.59ms
step:1126/1480 train_time:175817ms step_avg:157.54ms
step:1127/1480 train_time:175986ms step_avg:157.55ms
step:1128/1480 train_time:176158ms step_avg:157.57ms
step:1129/1480 train_time:176331ms step_avg:157.58ms
step:1130/1480 train_time:176502ms step_avg:157.59ms
step:1131/1480 train_time:176680ms step_avg:157.61ms
step:1132/1480 train_time:176846ms step_avg:157.62ms
step:1133/1480 train_time:177018ms step_avg:157.63ms
step:1134/1480 train_time:177189ms step_avg:157.64ms
step:1135/1480 train_time:177356ms step_avg:157.65ms
step:1136/1480 train_time:177528ms step_avg:157.66ms
step:1137/1480 train_time:177697ms step_avg:157.67ms
step:1138/1480 train_time:177870ms step_avg:157.69ms
step:1139/1480 train_time:178038ms step_avg:157.70ms
step:1140/1480 train_time:178207ms step_avg:157.71ms
step:1141/1480 train_time:178378ms step_avg:157.72ms
step:1142/1480 train_time:178545ms step_avg:157.73ms
step:1143/1480 train_time:178713ms step_avg:157.73ms
step:1144/1480 train_time:178881ms step_avg:157.74ms
step:1145/1480 train_time:179045ms step_avg:157.75ms
step:1146/1480 train_time:179215ms step_avg:157.76ms
step:1147/1480 train_time:179386ms step_avg:157.77ms
step:1148/1480 train_time:179553ms step_avg:157.78ms
step:1149/1480 train_time:179724ms step_avg:157.79ms
step:1150/1480 train_time:179894ms step_avg:157.80ms
step:1151/1480 train_time:180068ms step_avg:157.82ms
step:1152/1480 train_time:180240ms step_avg:157.83ms
step:1153/1480 train_time:180413ms step_avg:157.84ms
step:1154/1480 train_time:180580ms step_avg:157.85ms
step:1155/1480 train_time:180753ms step_avg:157.86ms
step:1156/1480 train_time:180932ms step_avg:157.88ms
step:1157/1480 train_time:181103ms step_avg:157.89ms
step:1158/1480 train_time:181270ms step_avg:157.90ms
step:1159/1480 train_time:181437ms step_avg:157.91ms
step:1160/1480 train_time:181605ms step_avg:157.92ms
step:1161/1480 train_time:181775ms step_avg:157.93ms
step:1162/1480 train_time:181945ms step_avg:157.94ms
step:1163/1480 train_time:182115ms step_avg:157.95ms
step:1164/1480 train_time:182284ms step_avg:157.96ms
step:1165/1480 train_time:182450ms step_avg:157.97ms
step:1166/1480 train_time:182619ms step_avg:157.98ms
step:1167/1480 train_time:182789ms step_avg:157.99ms
step:1168/1480 train_time:182955ms step_avg:157.99ms
step:1169/1480 train_time:183125ms step_avg:158.00ms
step:1170/1480 train_time:183295ms step_avg:158.01ms
step:1171/1480 train_time:183464ms step_avg:158.02ms
step:1172/1480 train_time:183630ms step_avg:158.03ms
step:1173/1480 train_time:183801ms step_avg:158.04ms
step:1174/1480 train_time:183984ms step_avg:158.06ms
step:1175/1480 train_time:184155ms step_avg:158.07ms
step:1176/1480 train_time:184327ms step_avg:158.09ms
step:1177/1480 train_time:184504ms step_avg:158.10ms
step:1178/1480 train_time:184672ms step_avg:158.11ms
step:1179/1480 train_time:184837ms step_avg:158.12ms
step:1180/1480 train_time:185016ms step_avg:158.13ms
step:1181/1480 train_time:185186ms step_avg:158.14ms
step:1182/1480 train_time:185354ms step_avg:158.15ms
step:1183/1480 train_time:185525ms step_avg:158.16ms
step:1184/1480 train_time:185692ms step_avg:158.17ms
step:1185/1480 train_time:185864ms step_avg:158.18ms
step:1186/1480 train_time:186035ms step_avg:158.19ms
step:1187/1480 train_time:186218ms step_avg:158.21ms
step:1188/1480 train_time:186385ms step_avg:158.22ms
step:1189/1480 train_time:186556ms step_avg:158.23ms
step:1190/1480 train_time:186724ms step_avg:158.24ms
step:1191/1480 train_time:186896ms step_avg:158.25ms
step:1192/1480 train_time:187062ms step_avg:158.26ms
step:1193/1480 train_time:187227ms step_avg:158.26ms
step:1194/1480 train_time:187395ms step_avg:158.27ms
step:1195/1480 train_time:187569ms step_avg:158.29ms
step:1196/1480 train_time:187751ms step_avg:158.31ms
step:1197/1480 train_time:187923ms step_avg:158.32ms
step:1198/1480 train_time:188108ms step_avg:158.34ms
step:1199/1480 train_time:188277ms step_avg:158.35ms
step:1200/1480 train_time:188447ms step_avg:158.36ms
step:1201/1480 train_time:188614ms step_avg:158.37ms
step:1202/1480 train_time:188795ms step_avg:158.39ms
step:1203/1480 train_time:188972ms step_avg:158.40ms
step:1204/1480 train_time:189146ms step_avg:158.41ms
step:1205/1480 train_time:189315ms step_avg:158.42ms
step:1206/1480 train_time:189483ms step_avg:158.43ms
step:1207/1480 train_time:189651ms step_avg:158.44ms
step:1208/1480 train_time:189818ms step_avg:158.45ms
step:1209/1480 train_time:189992ms step_avg:158.46ms
step:1210/1480 train_time:190170ms step_avg:158.47ms
step:1211/1480 train_time:190344ms step_avg:158.49ms
step:1212/1480 train_time:190516ms step_avg:158.50ms
step:1213/1480 train_time:190689ms step_avg:158.51ms
step:1214/1480 train_time:190868ms step_avg:158.53ms
step:1215/1480 train_time:191042ms step_avg:158.54ms
step:1216/1480 train_time:191212ms step_avg:158.55ms
step:1217/1480 train_time:191384ms step_avg:158.56ms
step:1218/1480 train_time:191553ms step_avg:158.57ms
step:1219/1480 train_time:191733ms step_avg:158.59ms
step:1220/1480 train_time:191904ms step_avg:158.60ms
step:1221/1480 train_time:192072ms step_avg:158.61ms
step:1222/1480 train_time:192239ms step_avg:158.61ms
step:1223/1480 train_time:192410ms step_avg:158.62ms
step:1224/1480 train_time:192587ms step_avg:158.64ms
step:1225/1480 train_time:192757ms step_avg:158.65ms
step:1226/1480 train_time:192932ms step_avg:158.66ms
step:1227/1480 train_time:193105ms step_avg:158.67ms
step:1228/1480 train_time:193274ms step_avg:158.68ms
step:1229/1480 train_time:193447ms step_avg:158.69ms
step:1230/1480 train_time:193625ms step_avg:158.71ms
step:1231/1480 train_time:193801ms step_avg:158.72ms
step:1232/1480 train_time:193975ms step_avg:158.74ms
step:1233/1480 train_time:194146ms step_avg:158.75ms
step:1234/1480 train_time:194315ms step_avg:158.75ms
step:1235/1480 train_time:194491ms step_avg:158.77ms
step:1236/1480 train_time:194658ms step_avg:158.77ms
step:1237/1480 train_time:194827ms step_avg:158.78ms
step:1238/1480 train_time:195013ms step_avg:158.81ms
step:1239/1480 train_time:195185ms step_avg:158.82ms
step:1240/1480 train_time:195354ms step_avg:158.82ms
step:1241/1480 train_time:195527ms step_avg:158.84ms
step:1242/1480 train_time:195697ms step_avg:158.84ms
step:1243/1480 train_time:195871ms step_avg:158.86ms
step:1244/1480 train_time:196037ms step_avg:158.86ms
step:1245/1480 train_time:196208ms step_avg:158.87ms
step:1246/1480 train_time:196377ms step_avg:158.88ms
step:1247/1480 train_time:196546ms step_avg:158.89ms
step:1248/1480 train_time:196714ms step_avg:158.90ms
step:1249/1480 train_time:196883ms step_avg:158.90ms
step:1250/1480 train_time:197052ms step_avg:158.91ms
step:1250/1480 val_loss:3.3363 train_time:197124ms step_avg:158.97ms
step:1251/1480 train_time:197234ms step_avg:158.93ms
step:1252/1480 train_time:197403ms step_avg:158.94ms
step:1253/1480 train_time:197572ms step_avg:158.95ms
step:1254/1480 train_time:197743ms step_avg:158.96ms
step:1255/1480 train_time:197932ms step_avg:158.98ms
step:1256/1480 train_time:198102ms step_avg:158.99ms
step:1257/1480 train_time:198272ms step_avg:159.00ms
step:1258/1480 train_time:198446ms step_avg:159.01ms
step:1259/1480 train_time:198617ms step_avg:159.02ms
step:1260/1480 train_time:198783ms step_avg:159.03ms
step:1261/1480 train_time:198957ms step_avg:159.04ms
step:1262/1480 train_time:199134ms step_avg:159.05ms
step:1263/1480 train_time:199307ms step_avg:159.06ms
step:1264/1480 train_time:199472ms step_avg:159.07ms
step:1265/1480 train_time:199640ms step_avg:159.08ms
step:1266/1480 train_time:199813ms step_avg:159.09ms
step:1267/1480 train_time:199985ms step_avg:159.10ms
step:1268/1480 train_time:200156ms step_avg:159.11ms
step:1269/1480 train_time:200332ms step_avg:159.12ms
step:1270/1480 train_time:200501ms step_avg:159.13ms
step:1271/1480 train_time:200671ms step_avg:159.14ms
step:1272/1480 train_time:200836ms step_avg:159.14ms
step:1273/1480 train_time:201007ms step_avg:159.15ms
step:1274/1480 train_time:201179ms step_avg:159.16ms
step:1275/1480 train_time:201347ms step_avg:159.17ms
step:1276/1480 train_time:201514ms step_avg:159.17ms
step:1277/1480 train_time:201684ms step_avg:159.18ms
step:1278/1480 train_time:201853ms step_avg:159.19ms
step:1279/1480 train_time:202023ms step_avg:159.20ms
step:1280/1480 train_time:202202ms step_avg:159.21ms
step:1281/1480 train_time:202371ms step_avg:159.22ms
step:1282/1480 train_time:202537ms step_avg:159.23ms
step:1283/1480 train_time:202708ms step_avg:159.24ms
step:1284/1480 train_time:202877ms step_avg:159.24ms
step:1285/1480 train_time:203046ms step_avg:159.25ms
step:1286/1480 train_time:203216ms step_avg:159.26ms
step:1287/1480 train_time:203387ms step_avg:159.27ms
step:1288/1480 train_time:203558ms step_avg:159.28ms
step:1289/1480 train_time:203740ms step_avg:159.30ms
step:1290/1480 train_time:203921ms step_avg:159.31ms
step:1291/1480 train_time:204094ms step_avg:159.32ms
step:1292/1480 train_time:204269ms step_avg:159.34ms
step:1293/1480 train_time:204443ms step_avg:159.35ms
step:1294/1480 train_time:204616ms step_avg:159.36ms
step:1295/1480 train_time:204787ms step_avg:159.37ms
step:1296/1480 train_time:204960ms step_avg:159.38ms
step:1297/1480 train_time:205132ms step_avg:159.39ms
step:1298/1480 train_time:205301ms step_avg:159.39ms
step:1299/1480 train_time:205472ms step_avg:159.40ms
step:1300/1480 train_time:205639ms step_avg:159.41ms
step:1301/1480 train_time:205807ms step_avg:159.42ms
step:1302/1480 train_time:205983ms step_avg:159.43ms
step:1303/1480 train_time:206159ms step_avg:159.44ms
step:1304/1480 train_time:206334ms step_avg:159.45ms
step:1305/1480 train_time:206502ms step_avg:159.46ms
step:1306/1480 train_time:206675ms step_avg:159.47ms
step:1307/1480 train_time:206842ms step_avg:159.48ms
step:1308/1480 train_time:207012ms step_avg:159.49ms
step:1309/1480 train_time:207184ms step_avg:159.49ms
step:1310/1480 train_time:207353ms step_avg:159.50ms
step:1311/1480 train_time:207520ms step_avg:159.51ms
step:1312/1480 train_time:207692ms step_avg:159.52ms
step:1313/1480 train_time:207859ms step_avg:159.52ms
step:1314/1480 train_time:208033ms step_avg:159.53ms
step:1315/1480 train_time:208203ms step_avg:159.54ms
step:1316/1480 train_time:208371ms step_avg:159.55ms
step:1317/1480 train_time:208542ms step_avg:159.56ms
step:1318/1480 train_time:208722ms step_avg:159.57ms
step:1319/1480 train_time:208897ms step_avg:159.58ms
step:1320/1480 train_time:209073ms step_avg:159.60ms
step:1321/1480 train_time:209245ms step_avg:159.61ms
step:1322/1480 train_time:209428ms step_avg:159.63ms
step:1323/1480 train_time:209599ms step_avg:159.63ms
step:1324/1480 train_time:209774ms step_avg:159.65ms
step:1325/1480 train_time:209955ms step_avg:159.66ms
step:1326/1480 train_time:210133ms step_avg:159.68ms
step:1327/1480 train_time:210302ms step_avg:159.68ms
step:1328/1480 train_time:210472ms step_avg:159.69ms
step:1329/1480 train_time:210668ms step_avg:159.72ms
step:1330/1480 train_time:210848ms step_avg:159.73ms
step:1331/1480 train_time:211018ms step_avg:159.74ms
step:1332/1480 train_time:211192ms step_avg:159.75ms
step:1333/1480 train_time:211368ms step_avg:159.76ms
step:1334/1480 train_time:211539ms step_avg:159.77ms
step:1335/1480 train_time:211709ms step_avg:159.78ms
step:1336/1480 train_time:211893ms step_avg:159.80ms
step:1337/1480 train_time:212068ms step_avg:159.81ms
step:1338/1480 train_time:212240ms step_avg:159.82ms
step:1339/1480 train_time:212415ms step_avg:159.83ms
step:1340/1480 train_time:212587ms step_avg:159.84ms
step:1341/1480 train_time:212756ms step_avg:159.85ms
step:1342/1480 train_time:212930ms step_avg:159.86ms
step:1343/1480 train_time:213099ms step_avg:159.86ms
step:1344/1480 train_time:213272ms step_avg:159.87ms
step:1345/1480 train_time:213451ms step_avg:159.89ms
step:1346/1480 train_time:213620ms step_avg:159.90ms
step:1347/1480 train_time:213790ms step_avg:159.90ms
step:1348/1480 train_time:213959ms step_avg:159.91ms
step:1349/1480 train_time:214130ms step_avg:159.92ms
step:1350/1480 train_time:214303ms step_avg:159.93ms
step:1351/1480 train_time:214473ms step_avg:159.94ms
step:1352/1480 train_time:214643ms step_avg:159.94ms
step:1353/1480 train_time:214819ms step_avg:159.95ms
step:1354/1480 train_time:214990ms step_avg:159.96ms
step:1355/1480 train_time:215157ms step_avg:159.97ms
step:1356/1480 train_time:215331ms step_avg:159.98ms
step:1357/1480 train_time:215503ms step_avg:159.99ms
step:1358/1480 train_time:215674ms step_avg:160.00ms
step:1359/1480 train_time:215847ms step_avg:160.01ms
step:1360/1480 train_time:216022ms step_avg:160.02ms
step:1361/1480 train_time:216201ms step_avg:160.03ms
step:1362/1480 train_time:216376ms step_avg:160.04ms
step:1363/1480 train_time:216556ms step_avg:160.06ms
step:1364/1480 train_time:216725ms step_avg:160.06ms
step:1365/1480 train_time:216893ms step_avg:160.07ms
step:1366/1480 train_time:217064ms step_avg:160.08ms
step:1367/1480 train_time:217235ms step_avg:160.08ms
step:1368/1480 train_time:217408ms step_avg:160.09ms
step:1369/1480 train_time:217589ms step_avg:160.11ms
step:1370/1480 train_time:217768ms step_avg:160.12ms
step:1371/1480 train_time:217940ms step_avg:160.13ms
step:1372/1480 train_time:218116ms step_avg:160.14ms
step:1373/1480 train_time:218286ms step_avg:160.15ms
step:1374/1480 train_time:218461ms step_avg:160.16ms
step:1375/1480 train_time:218633ms step_avg:160.17ms
step:1375/1480 val_loss:3.2979 train_time:218700ms step_avg:160.22ms
step:1376/1480 train_time:218807ms step_avg:160.18ms
step:1377/1480 train_time:218979ms step_avg:160.19ms
step:1378/1480 train_time:219149ms step_avg:160.20ms
step:1379/1480 train_time:219325ms step_avg:160.21ms
step:1380/1480 train_time:219500ms step_avg:160.22ms
step:1381/1480 train_time:219683ms step_avg:160.24ms
step:1382/1480 train_time:219853ms step_avg:160.24ms
step:1383/1480 train_time:220026ms step_avg:160.25ms
step:1384/1480 train_time:220205ms step_avg:160.27ms
step:1385/1480 train_time:220371ms step_avg:160.27ms
step:1386/1480 train_time:220542ms step_avg:160.28ms
step:1387/1480 train_time:220714ms step_avg:160.29ms
step:1388/1480 train_time:220883ms step_avg:160.29ms
step:1389/1480 train_time:221055ms step_avg:160.30ms
step:1390/1480 train_time:221223ms step_avg:160.31ms
step:1391/1480 train_time:221392ms step_avg:160.31ms
step:1392/1480 train_time:221565ms step_avg:160.32ms
step:1393/1480 train_time:221736ms step_avg:160.33ms
step:1394/1480 train_time:221907ms step_avg:160.34ms
step:1395/1480 train_time:222075ms step_avg:160.34ms
step:1396/1480 train_time:222245ms step_avg:160.35ms
step:1397/1480 train_time:222411ms step_avg:160.35ms
step:1398/1480 train_time:222578ms step_avg:160.36ms
step:1399/1480 train_time:222748ms step_avg:160.37ms
step:1400/1480 train_time:222926ms step_avg:160.38ms
step:1401/1480 train_time:223091ms step_avg:160.38ms
step:1402/1480 train_time:223263ms step_avg:160.39ms
step:1403/1480 train_time:223440ms step_avg:160.40ms
step:1404/1480 train_time:223611ms step_avg:160.41ms
step:1405/1480 train_time:223786ms step_avg:160.42ms
step:1406/1480 train_time:223960ms step_avg:160.43ms
step:1407/1480 train_time:224129ms step_avg:160.44ms
step:1408/1480 train_time:224298ms step_avg:160.44ms
step:1409/1480 train_time:224481ms step_avg:160.46ms
step:1410/1480 train_time:224649ms step_avg:160.46ms
step:1411/1480 train_time:224817ms step_avg:160.47ms
step:1412/1480 train_time:224987ms step_avg:160.48ms
step:1413/1480 train_time:225158ms step_avg:160.48ms
step:1414/1480 train_time:225330ms step_avg:160.49ms
step:1415/1480 train_time:225506ms step_avg:160.50ms
step:1416/1480 train_time:225692ms step_avg:160.52ms
step:1417/1480 train_time:225866ms step_avg:160.53ms
step:1418/1480 train_time:226037ms step_avg:160.54ms
step:1419/1480 train_time:226211ms step_avg:160.55ms
step:1420/1480 train_time:226385ms step_avg:160.56ms
step:1421/1480 train_time:226559ms step_avg:160.57ms
step:1422/1480 train_time:226731ms step_avg:160.57ms
step:1423/1480 train_time:226901ms step_avg:160.58ms
step:1424/1480 train_time:227078ms step_avg:160.59ms
step:1425/1480 train_time:227256ms step_avg:160.60ms
step:1426/1480 train_time:227427ms step_avg:160.61ms
step:1427/1480 train_time:227602ms step_avg:160.62ms
step:1428/1480 train_time:227772ms step_avg:160.63ms
step:1429/1480 train_time:227942ms step_avg:160.64ms
step:1430/1480 train_time:228117ms step_avg:160.65ms
step:1431/1480 train_time:228292ms step_avg:160.66ms
step:1432/1480 train_time:228469ms step_avg:160.67ms
step:1433/1480 train_time:228649ms step_avg:160.68ms
step:1434/1480 train_time:228829ms step_avg:160.69ms
step:1435/1480 train_time:229004ms step_avg:160.70ms
step:1436/1480 train_time:229179ms step_avg:160.71ms
step:1437/1480 train_time:229348ms step_avg:160.72ms
step:1438/1480 train_time:229517ms step_avg:160.73ms
step:1439/1480 train_time:229691ms step_avg:160.74ms
step:1440/1480 train_time:229862ms step_avg:160.74ms
step:1441/1480 train_time:230032ms step_avg:160.75ms
step:1442/1480 train_time:230209ms step_avg:160.76ms
step:1443/1480 train_time:230398ms step_avg:160.78ms
step:1444/1480 train_time:230569ms step_avg:160.79ms
step:1445/1480 train_time:230742ms step_avg:160.80ms
step:1446/1480 train_time:230917ms step_avg:160.81ms
step:1447/1480 train_time:231094ms step_avg:160.82ms
step:1448/1480 train_time:231266ms step_avg:160.82ms
step:1449/1480 train_time:231440ms step_avg:160.83ms
step:1450/1480 train_time:231611ms step_avg:160.84ms
step:1451/1480 train_time:231781ms step_avg:160.85ms
step:1452/1480 train_time:231953ms step_avg:160.86ms
step:1453/1480 train_time:232124ms step_avg:160.86ms
step:1454/1480 train_time:232295ms step_avg:160.87ms
step:1455/1480 train_time:232475ms step_avg:160.88ms
step:1456/1480 train_time:232649ms step_avg:160.89ms
step:1457/1480 train_time:232820ms step_avg:160.90ms
step:1458/1480 train_time:232989ms step_avg:160.90ms
step:1459/1480 train_time:233165ms step_avg:160.91ms
step:1460/1480 train_time:233338ms step_avg:160.92ms
step:1461/1480 train_time:233512ms step_avg:160.93ms
step:1462/1480 train_time:233683ms step_avg:160.94ms
step:1463/1480 train_time:233859ms step_avg:160.95ms
step:1464/1480 train_time:234032ms step_avg:160.96ms
step:1465/1480 train_time:234205ms step_avg:160.97ms
step:1466/1480 train_time:234376ms step_avg:160.97ms
step:1467/1480 train_time:234551ms step_avg:160.98ms
step:1468/1480 train_time:234723ms step_avg:160.99ms
step:1469/1480 train_time:234895ms step_avg:161.00ms
step:1470/1480 train_time:235074ms step_avg:161.01ms
step:1471/1480 train_time:235262ms step_avg:161.03ms
step:1472/1480 train_time:235445ms step_avg:161.04ms
step:1473/1480 train_time:235616ms step_avg:161.05ms
step:1474/1480 train_time:235796ms step_avg:161.06ms
step:1475/1480 train_time:235975ms step_avg:161.08ms
step:1476/1480 train_time:236148ms step_avg:161.08ms
step:1477/1480 train_time:236330ms step_avg:161.10ms
step:1478/1480 train_time:236512ms step_avg:161.11ms
step:1479/1480 train_time:236685ms step_avg:161.12ms
step:1480/1480 train_time:236858ms step_avg:161.13ms
step:1480/1480 val_loss:3.2791 train_time:236929ms step_avg:161.18ms