bbc9dd90-96c5-480b-984c-594145555821.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
from dataclasses import dataclass
from pathlib import Path

import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
# Use of FlexAttention contributed by @KoszarskyB
from torch.nn.attention.flex_attention import BlockMask, flex_attention

# -----------------------------------------------------------------------------
# Muon optimizer

def zeropower_via_svd(G, steps=None):
    U, S, V = G.svd()
    return U @ V.T

@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert len(G.shape) == 2
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    X /= (X.norm() + eps) # ensure top singular value <= 1
    if G.size(0) > G.size(1):
        X = X.T
    for _ in range(steps):
        A = X @ X.T
        B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    if G.size(0) > G.size(1):
        X = X.T
    return X

zeropower_backends = dict(svd=zeropower_via_svd, newtonschulz5=zeropower_via_newtonschulz5)

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven't tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        backend: The chosen backend for the orthogonalization step. (recommended: 'newtonschulz5')
        backend_steps: The number of iteration steps to use in the backend, if it is iterative.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True,
                 backend='newtonschulz5', backend_steps=5):
        self.num_process = int(os.environ['WORLD_SIZE'])
        self.rank = int(os.environ["RANK"])
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, backend=backend, backend_steps=backend_steps)
        params: "list[torch.Tensor]" = list(params)
        assert all(isinstance(p, torch.Tensor) for p in params)
        sizes = {p.numel() for p in params}
        param_groups = [
            {
                "params": [p for p in params if p.numel() == size],
                "update_buffer": [
                    torch.empty(size, device="cuda", dtype=torch.bfloat16)
                    for _ in range(self.num_process)
                ],
            }
            for size in sizes
        ]
        super().__init__(param_groups, defaults)

    def step(self):
        for group in self.param_groups:
            lr: float = group["lr"]
            momentum: float = group["momentum"]
            nesterov: bool = group["nesterov"]
            zeropower_backend = zeropower_backends[group["backend"]]
            backend_steps: int = group["backend_steps"]
            update_buffers: "list[torch.Tensor]" = group["update_buffer"]
            # generate weight updates in distributed fashion
            params: "list[torch.Tensor]" = group["params"]
            assert len(params) % self.num_process == 0
            handle = None
            params_world = None
            def update_prev():
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffers):
                    p_world.data.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.num_process]:
                p = params[base_i + self.rank]
                g = p.grad
                assert g is not None
                state = self.state[p] 
                if "momentum_buffer" not in state:
                    state["momentum_buffer"] = torch.zeros_like(g)
                buf: torch.Tensor = state["momentum_buffer"]
                buf.lerp_(g, 1 - momentum)
                g = g.lerp_(buf, momentum) if nesterov else buf
                g = zeropower_backend(g, steps=backend_steps).flatten()
                update_prev()
                handle = dist.all_gather(update_buffers, g, async_op=True)
                params_world = params[base_i : base_i + self.num_process]
            update_prev()


# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):

    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=False)

    def forward(self, x):
        return F.linear(x, self.weight.to(x.dtype))

class Rotary(torch.nn.Module):

    def __init__(self, dim, base=10000):
        super().__init__()
        self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None

    def forward(self, x):
        seq_len = x.shape[1]
        if seq_len != self.seq_len_cached:
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.outer(t, self.inv_freq)
            self.seq_len_cached = seq_len
            self.cos_cached = freqs.cos()
            self.sin_cached = freqs.sin()
        cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
        # apply_rotary_emb(x, cos, sin)
        x1, x2 = x.chunk(2, dim=3)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x)

class CausalSelfAttention(nn.Module):

    def __init__(self, dim, n_head):
        super().__init__()
        assert dim % n_head == 0
        self.n_head = n_head
        self.c_q = CastedLinear(dim, dim)
        self.c_k = CastedLinear(dim, dim)
        self.c_v = CastedLinear(dim, dim)
        # value residual lambda
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5])) # @Grad62304977
        # rotary embeddings
        self.rotary = Rotary(dim // n_head) # dim // n_head = head_dim
        # output projection
        self.c_proj = CastedLinear(dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor, vi: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q: torch.Tensor = self.c_q(x).view(B, T, self.n_head, -1)
        k: torch.Tensor = self.c_k(x).view(B, T, self.n_head, -1)
        v: torch.Tensor = self.c_v(x).view(B, T, self.n_head, -1)
        v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @Grad62304977
        q, k = norm(q), norm(k) # QK norm suggested by @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask)
        y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):

    def __init__(self, dim: int):
        super().__init__()
        self.c_fc   = CastedLinear(dim, 4 * dim)
        self.c_proj = CastedLinear(4 * dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):

    def __init__(self, config):
        super().__init__()
        self.attn = CausalSelfAttention(config.n_embd, config.n_head)
        self.mlp = MLP(config.n_embd)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x: torch.Tensor, vi: torch.Tensor, x0: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        x = x + self.attn(norm(x), vi, block_mask)
        x = x + self.mlp(norm(x))
        return x

# -----------------------------------------------------------------------------
# The main GPT-2 model

@dataclass
class GPTConfig:
    vocab_size : int = 50304
    n_layer : int = 12
    n_head : int = 6 # head dim 128 suggested by @Grad62304977
    n_embd : int = 768
    lm_head_softcap : int = 30

class GPT(nn.Module):

    def __init__(self, config: GPTConfig):
        super().__init__()
        self.n_layer = config.n_layer
        self.lm_head_softcap = config.lm_head_softcap

        # U-net design by @brendanh0gan
        self.num_encoder_layers = config.n_layer // 2 # Half of the layers for encoder
        self.num_decoder_layers = config.n_layer - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))

        self.transformer = nn.ModuleDict(dict(
            wte = nn.Embedding(config.vocab_size, config.n_embd),
            # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
            # U-net structure on token value embeddings by @leloykun
            vte = nn.Embedding(config.vocab_size, config.n_embd*self.num_encoder_layers),
            h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
        ))
        self.lm_head = CastedLinear(config.n_embd, config.vocab_size)
        self.lm_head.weight.data.zero_() # @Grad62304977

    def forward(self, idx: torch.Tensor, target: torch.Tensor, sliding_window: torch.Tensor) -> torch.Tensor:
        BLOCK_SIZE = 128
        assert idx.ndim == 1
        docs = (idx == 50256).cumsum(0)
        docs_low = docs.reshape(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.reshape(-1, BLOCK_SIZE)[:, -1].contiguous()
        def document_sliding_window_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            window_mask = q_idx - kv_idx < sliding_window
            return causal_mask & document_mask & window_mask

        S = len(idx)
        def create_sliding_window_causal_mask(S: int, sliding_window: torch.Tensor):
            kv_idx = block_idx = torch.arange(S // BLOCK_SIZE, dtype=torch.int32, device="cuda")
            q_idx = block_idx[:, None]
            causal_mask = q_idx >= kv_idx
            document_mask = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
            window_mask = q_idx - kv_idx < ((sliding_window + BLOCK_SIZE - 1) // BLOCK_SIZE)
            dense_mask = causal_mask & document_mask & window_mask
            dense_mask = dense_mask.to(torch.int32)
            num_blocks = dense_mask.sum(dim=-1).to(torch.int32)
            indices = torch.argsort(dense_mask, dim=-1, descending=True, stable=True).to(torch.int32)
            num_blocks = num_blocks[None, None, :].contiguous()
            indices = indices[None, None, :].contiguous()
            return BlockMask.from_kv_blocks(num_blocks, indices, BLOCK_SIZE=BLOCK_SIZE, mask_mod=document_sliding_window_causal)
        block_mask = create_sliding_window_causal_mask(S, sliding_window)

        # forward the GPT model itself
        x = self.transformer.wte(idx[None]) # token embeddings of shape (b, t, n_embd)
        x = norm(x) # @Grad62304977
        x0 = x
        vi = self.transformer.vte(idx[None]).chunk(self.num_encoder_layers, dim=-1)

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        for i in range(self.num_encoder_layers):
            x = self.transformer.h[i](x, vi[i], x0, block_mask)
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            # U-net structure on token value embeddings by @leloykun
            x = self.transformer.h[self.num_encoder_layers + i](x, vi[self.num_encoder_layers-1-i], x0, block_mask)

        x = norm(x)
        logits = self.lm_head(x)
        logits = self.lm_head_softcap * torch.tanh(logits / self.lm_head_softcap) # @Grad62304977
        logits = logits.float()
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), target.view(-1))
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _peek_data_shard(file: Path):
    # only reads the header, returns header data
    # header is 256 int32
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    return int(header[2]) # number of tokens (claimed)

def _load_data_shard(file: Path, ntok: int):
    with file.open("rb") as f:
        tokens = torch.empty(ntok, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy())
        assert nbytes == 2 * ntok, "number of tokens read does not match header?"
    return tokens

class DistributedDataLoader:
    def __init__(self, filename_pattern, T, process_rank, num_processes):
        self.process_rank = process_rank
        self.num_processes = num_processes
        self.T = T

        # glob files that match the pattern
        self.files = sorted(Path.cwd().glob(filename_pattern))
        assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"

        # load and validate all data shards, count number of tokens in total
        self.ntoks = [_peek_data_shard(file) for file in self.files]
        assert min(self.ntoks) >= num_processes * T + 1
        self.ntok_total = sum(self.ntoks)

        self.reset()

    def reset(self):
        self.current_shard = -1
        self.advance()

    def advance(self): # advance to next data shard
        self.current_shard = (self.current_shard + 1) % len(self.files)
        self.current_position = self.process_rank * self.T
        self.tokens = _load_data_shard(self.files[self.current_shard], self.ntoks[self.current_shard])

    def next_batch(self):
        batch_size = self.T * self.num_processes
        buf = self.tokens[self.current_position:self.current_position+self.T+1]
        # host side async is sufficient;
        # no performance improvement was observed when introducing a separate stream.
        x = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
        y = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
        # advance current position and load next shard if necessary
        self.current_position += batch_size
        if self.current_position + batch_size + 1 >= len(self.tokens):
            self.advance()
        return x, y

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data hyperparams
    input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
    input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
    # optimization hyperparams
    batch_size : int = 8 # batch size, in sequences, across all devices
    sequence_length : int = 64*1024 # sequence length, in tokens
    num_iterations : int = 1480 # number of iterations to run
    warmup_iters : int = 0
    cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
    weight_decay : float = 0
    # evaluation and logging hyperparams
    val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
    val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()

# set up DDP (distributed data parallel). torchrun sets this env variable
assert torch.cuda.is_available()
dist.init_process_group(backend='nccl')
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
device = f'cuda:{ddp_local_rank}'
torch.cuda.set_device(device)
print(f"using device: {device}")
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = str(uuid.uuid4())
    logdir = 'logs/%s/' % run_id
    # os.makedirs(logdir, exist_ok=True)
    logfile = 'logs/%s.txt' % run_id
    # create the log file
    with open(logfile, "w") as f:
        # begin the log by printing this file (the Python code)
        f.write(code)
        f.write('='*100 + '\n')
def print0(s, logonly=False):
    if master_process:
        with open(logfile, "a") as f:
            if not logonly:
                print(s)
            f.write(s+'\n')
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)

# convenience variables
T = args.sequence_length
# calculate the number of steps to take in the val loop.
assert args.val_tokens % (T * ddp_world_size) == 0
val_steps = args.val_tokens // (T * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size
assert train_accumulation_steps == 1

# load tokens
train_loader = DistributedDataLoader(args.input_bin, T, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, T, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.ntok_total} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.ntok_total} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
x, y = train_loader.next_batch()

# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, n_layer=12, n_head=6, n_embd=768))
model = model.cuda().bfloat16()
for m in model.modules():
    if isinstance(m, CastedLinear):
        m.float()
if hasattr(config, "coordinate_descent_tuning"):
    config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank])
raw_model = model.module # always contains the "raw" unwrapped model

# init the optimizer(s)
optimizer1 = torch.optim.Adam([raw_model.transformer.wte.weight, raw_model.transformer.vte.weight], lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.transformer.h.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
    assert it <= args.num_iterations
    # 1) linear warmup for warmup_iters steps
    if it < args.warmup_iters:
        return (it+1) / args.warmup_iters
    # 2) constant lr for a while
    elif it < args.num_iterations - args.cooldown_iters:
        return 1.0
    # 3) linear cooldown
    else:
        decay_ratio = (args.num_iterations - it) / args.cooldown_iters
        return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]

sliding_window_size = torch.tensor(64, dtype=torch.int32, device="cuda")
sw_size_prev = 64
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
    last_step = (step == args.num_iterations)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Set the sliding window size for the current step, in chunks of 64. By @fernbear.bsky.social
    sw_size =  64 * int((64 + (1792 - 64) * step / args.num_iterations) // 64)
    if sw_size != sw_size_prev:
        sliding_window_size.copy_(sw_size, non_blocking=True)
        sw_size_prev = sw_size

    # once in a while evaluate the validation dataset
    if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # run validation batches
        model.eval()
        val_loader.reset()
        val_loss = 0.0
        for _ in range(val_steps):
            with torch.no_grad():
                x_val, y_val = val_loader.next_batch()
                val_loss += model(x_val, y_val, sliding_window=sliding_window_size)
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        val_loss /= val_steps
        # log val loss to console and to logfile
        print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # save the state of the training process
        log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
        # torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    # bit confusing: we want to make sure to eval on 0th iteration
    # but also after the very last iteration. so we loop for step <= num_iterations
    # instead of just < num_iterations (one extra due to <=), only to do
    # the validation/sampling one last time, and then we break right here as we're done.
    if last_step:
        break

    # --------------- TRAINING SECTION BEGIN -----------------
    model.train()
    loss = model(x, y, sliding_window=sliding_window_size)
    loss.backward()
    del loss
    # advance the dataset for the next batch
    x, y = train_loader.next_batch()
    # momentum warmup for Muon
    frac = min(step/300, 1)
    for group in optimizer3.param_groups:
        group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # --------------- TRAINING SECTION END -------------------
    # everything that follows now is just diagnostics, prints, logging, etc.
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")

if master_process:
    print(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")

# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()
====================================================================================================
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Sun Dec  8 14:01:11 2024       
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.6     |
|-----------------------------------------+----------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
|                                         |                      |               MIG M. |
|=========================================+======================+======================|
|   0  NVIDIA H100 80GB HBM3          On  | 00000000:65:02.0 Off |                    0 |
| N/A   36C    P0              74W / 700W |      7MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  | 00000000:67:02.0 Off |                    0 |
| N/A   45C    P0              92W / 700W |     26MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  | 00000000:69:02.0 Off |                    0 |
| N/A   45C    P0             122W / 700W |    533MiB / 81559MiB |      1%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  | 00000000:6B:02.0 Off |                    0 |
| N/A   39C    P0             109W / 700W |    119MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  | 00000000:6F:02.0 Off |                    0 |
| N/A   38C    P0             117W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  | 00000000:71:02.0 Off |                    0 |
| N/A   45C    P0             121W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  | 00000000:73:02.0 Off |                    0 |
| N/A   45C    P0             127W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  | 00000000:75:02.0 Off |                    0 |
| N/A   38C    P0             111W / 700W |     35MiB / 81559MiB |      1%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
                                                                                         
+---------------------------------------------------------------------------------------+
| Processes:                                                                            |
|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
|        ID   ID                                                             Usage      |
|=======================================================================================|
+---------------------------------------------------------------------------------------+

====================================================================================================
Training DataLoader: total number of tokens: 3200000000 across 32 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:23608ms step_avg:nanms
step:2/1480 train_time:23704ms step_avg:nanms
step:3/1480 train_time:23842ms step_avg:nanms
step:4/1480 train_time:23983ms step_avg:nanms
step:5/1480 train_time:24125ms step_avg:nanms
step:6/1480 train_time:24267ms step_avg:nanms
step:7/1480 train_time:24408ms step_avg:nanms
step:8/1480 train_time:24552ms step_avg:nanms
step:9/1480 train_time:24696ms step_avg:nanms
step:10/1480 train_time:24838ms step_avg:nanms
step:11/1480 train_time:143ms step_avg:nanms
step:12/1480 train_time:285ms step_avg:nanms
step:13/1480 train_time:429ms step_avg:142.87ms
step:14/1480 train_time:571ms step_avg:142.86ms
step:15/1480 train_time:715ms step_avg:142.94ms
step:16/1480 train_time:859ms step_avg:143.16ms
step:17/1480 train_time:1001ms step_avg:143.01ms
step:18/1480 train_time:1143ms step_avg:142.83ms
step:19/1480 train_time:1286ms step_avg:142.86ms
step:20/1480 train_time:1428ms step_avg:142.84ms
step:21/1480 train_time:1572ms step_avg:142.87ms
step:22/1480 train_time:1715ms step_avg:142.93ms
step:23/1480 train_time:1857ms step_avg:142.86ms
step:24/1480 train_time:2000ms step_avg:142.82ms
step:25/1480 train_time:2141ms step_avg:142.72ms
step:26/1480 train_time:2283ms step_avg:142.69ms
step:27/1480 train_time:2426ms step_avg:142.69ms
step:28/1480 train_time:2568ms step_avg:142.68ms
step:29/1480 train_time:2712ms step_avg:142.74ms
step:30/1480 train_time:2855ms step_avg:142.74ms
step:31/1480 train_time:2999ms step_avg:142.79ms
step:32/1480 train_time:3141ms step_avg:142.80ms
step:33/1480 train_time:3285ms step_avg:142.81ms
step:34/1480 train_time:3428ms step_avg:142.82ms
step:35/1480 train_time:3571ms step_avg:142.83ms
step:36/1480 train_time:3714ms step_avg:142.85ms
step:37/1480 train_time:3857ms step_avg:142.85ms
step:38/1480 train_time:3999ms step_avg:142.83ms
step:39/1480 train_time:4140ms step_avg:142.77ms
step:40/1480 train_time:4282ms step_avg:142.73ms
step:41/1480 train_time:4425ms step_avg:142.75ms
step:42/1480 train_time:4568ms step_avg:142.76ms
step:43/1480 train_time:4712ms step_avg:142.79ms
step:44/1480 train_time:4855ms step_avg:142.79ms
step:45/1480 train_time:4998ms step_avg:142.80ms
step:46/1480 train_time:5140ms step_avg:142.78ms
step:47/1480 train_time:5282ms step_avg:142.76ms
step:48/1480 train_time:5425ms step_avg:142.78ms
step:49/1480 train_time:5569ms step_avg:142.80ms
step:50/1480 train_time:5712ms step_avg:142.81ms
step:51/1480 train_time:5854ms step_avg:142.79ms
step:52/1480 train_time:5996ms step_avg:142.77ms
step:53/1480 train_time:6138ms step_avg:142.75ms
step:54/1480 train_time:6280ms step_avg:142.73ms
step:55/1480 train_time:6421ms step_avg:142.70ms
step:56/1480 train_time:6564ms step_avg:142.69ms
step:57/1480 train_time:6707ms step_avg:142.70ms
step:58/1480 train_time:6849ms step_avg:142.68ms
step:59/1480 train_time:6992ms step_avg:142.69ms
step:60/1480 train_time:7134ms step_avg:142.68ms
step:61/1480 train_time:7275ms step_avg:142.65ms
step:62/1480 train_time:7416ms step_avg:142.62ms
step:63/1480 train_time:7558ms step_avg:142.60ms
step:64/1480 train_time:7702ms step_avg:142.63ms
step:65/1480 train_time:7845ms step_avg:142.64ms
step:66/1480 train_time:7989ms step_avg:142.66ms
step:67/1480 train_time:8132ms step_avg:142.67ms
step:68/1480 train_time:8274ms step_avg:142.66ms
step:69/1480 train_time:8417ms step_avg:142.66ms
step:70/1480 train_time:8559ms step_avg:142.64ms
step:71/1480 train_time:8700ms step_avg:142.63ms
step:72/1480 train_time:8841ms step_avg:142.60ms
step:73/1480 train_time:8985ms step_avg:142.62ms
step:74/1480 train_time:9128ms step_avg:142.62ms
step:75/1480 train_time:9270ms step_avg:142.62ms
step:76/1480 train_time:9414ms step_avg:142.63ms
step:77/1480 train_time:9555ms step_avg:142.62ms
step:78/1480 train_time:9697ms step_avg:142.60ms
step:79/1480 train_time:9838ms step_avg:142.59ms
step:80/1480 train_time:9981ms step_avg:142.58ms
step:81/1480 train_time:10123ms step_avg:142.57ms
step:82/1480 train_time:10266ms step_avg:142.59ms
step:83/1480 train_time:10411ms step_avg:142.61ms
step:84/1480 train_time:10554ms step_avg:142.62ms
step:85/1480 train_time:10697ms step_avg:142.63ms
step:86/1480 train_time:10838ms step_avg:142.61ms
step:87/1480 train_time:10981ms step_avg:142.61ms
step:88/1480 train_time:11124ms step_avg:142.61ms
step:89/1480 train_time:11267ms step_avg:142.62ms
step:90/1480 train_time:11412ms step_avg:142.65ms
step:91/1480 train_time:11555ms step_avg:142.65ms
step:92/1480 train_time:11696ms step_avg:142.64ms
step:93/1480 train_time:11838ms step_avg:142.63ms
step:94/1480 train_time:11979ms step_avg:142.61ms
step:95/1480 train_time:12122ms step_avg:142.61ms
step:96/1480 train_time:12265ms step_avg:142.61ms
step:97/1480 train_time:12407ms step_avg:142.61ms
step:98/1480 train_time:12550ms step_avg:142.62ms
step:99/1480 train_time:12693ms step_avg:142.62ms
step:100/1480 train_time:12835ms step_avg:142.61ms
step:101/1480 train_time:12976ms step_avg:142.59ms
step:102/1480 train_time:13119ms step_avg:142.59ms
step:103/1480 train_time:13260ms step_avg:142.58ms
step:104/1480 train_time:13404ms step_avg:142.59ms
step:105/1480 train_time:13549ms step_avg:142.63ms
step:106/1480 train_time:13691ms step_avg:142.61ms
step:107/1480 train_time:13833ms step_avg:142.60ms
step:108/1480 train_time:13973ms step_avg:142.59ms
step:109/1480 train_time:14116ms step_avg:142.58ms
step:110/1480 train_time:14258ms step_avg:142.58ms
step:111/1480 train_time:14403ms step_avg:142.60ms
step:112/1480 train_time:14550ms step_avg:142.64ms
step:113/1480 train_time:14697ms step_avg:142.69ms
step:114/1480 train_time:14844ms step_avg:142.73ms
step:115/1480 train_time:14991ms step_avg:142.77ms
step:116/1480 train_time:15139ms step_avg:142.82ms
step:117/1480 train_time:15286ms step_avg:142.86ms
step:118/1480 train_time:15434ms step_avg:142.91ms
step:119/1480 train_time:15580ms step_avg:142.93ms
step:120/1480 train_time:15727ms step_avg:142.97ms
step:121/1480 train_time:15874ms step_avg:143.01ms
step:122/1480 train_time:16020ms step_avg:143.04ms
step:123/1480 train_time:16168ms step_avg:143.08ms
step:124/1480 train_time:16315ms step_avg:143.11ms
step:125/1480 train_time:16460ms step_avg:143.13ms
step:125/1480 val_loss:4.3968 train_time:16519ms step_avg:143.64ms
step:126/1480 train_time:16614ms step_avg:143.23ms
step:127/1480 train_time:16761ms step_avg:143.25ms
step:128/1480 train_time:16909ms step_avg:143.29ms
step:129/1480 train_time:17054ms step_avg:143.31ms
step:130/1480 train_time:17199ms step_avg:143.33ms
step:131/1480 train_time:17346ms step_avg:143.35ms
step:132/1480 train_time:17492ms step_avg:143.38ms
step:133/1480 train_time:17639ms step_avg:143.40ms
step:134/1480 train_time:17786ms step_avg:143.43ms
step:135/1480 train_time:17933ms step_avg:143.46ms
step:136/1480 train_time:18079ms step_avg:143.49ms
step:137/1480 train_time:18226ms step_avg:143.51ms
step:138/1480 train_time:18373ms step_avg:143.54ms
step:139/1480 train_time:18518ms step_avg:143.55ms
step:140/1480 train_time:18666ms step_avg:143.58ms
step:141/1480 train_time:18812ms step_avg:143.61ms
step:142/1480 train_time:18960ms step_avg:143.64ms
step:143/1480 train_time:19107ms step_avg:143.66ms
step:144/1480 train_time:19254ms step_avg:143.69ms
step:145/1480 train_time:19401ms step_avg:143.71ms
step:146/1480 train_time:19548ms step_avg:143.73ms
step:147/1480 train_time:19694ms step_avg:143.75ms
step:148/1480 train_time:19840ms step_avg:143.77ms
step:149/1480 train_time:19987ms step_avg:143.79ms
step:150/1480 train_time:20134ms step_avg:143.82ms
step:151/1480 train_time:20280ms step_avg:143.83ms
step:152/1480 train_time:20427ms step_avg:143.86ms
step:153/1480 train_time:20574ms step_avg:143.87ms
step:154/1480 train_time:20719ms step_avg:143.88ms
step:155/1480 train_time:20867ms step_avg:143.91ms
step:156/1480 train_time:21014ms step_avg:143.93ms
step:157/1480 train_time:21162ms step_avg:143.96ms
step:158/1480 train_time:21309ms step_avg:143.98ms
step:159/1480 train_time:21456ms step_avg:144.00ms
step:160/1480 train_time:21602ms step_avg:144.01ms
step:161/1480 train_time:21749ms step_avg:144.03ms
step:162/1480 train_time:21895ms step_avg:144.05ms
step:163/1480 train_time:22042ms step_avg:144.07ms
step:164/1480 train_time:22189ms step_avg:144.09ms
step:165/1480 train_time:22335ms step_avg:144.10ms
step:166/1480 train_time:22482ms step_avg:144.11ms
step:167/1480 train_time:22629ms step_avg:144.13ms
step:168/1480 train_time:22775ms step_avg:144.14ms
step:169/1480 train_time:22921ms step_avg:144.16ms
step:170/1480 train_time:23069ms step_avg:144.18ms
step:171/1480 train_time:23215ms step_avg:144.19ms
step:172/1480 train_time:23362ms step_avg:144.21ms
step:173/1480 train_time:23509ms step_avg:144.23ms
step:174/1480 train_time:23655ms step_avg:144.24ms
step:175/1480 train_time:23802ms step_avg:144.25ms
step:176/1480 train_time:23949ms step_avg:144.27ms
step:177/1480 train_time:24096ms step_avg:144.29ms
step:178/1480 train_time:24243ms step_avg:144.30ms
step:179/1480 train_time:24390ms step_avg:144.32ms
step:180/1480 train_time:24536ms step_avg:144.33ms
step:181/1480 train_time:24684ms step_avg:144.35ms
step:182/1480 train_time:24831ms step_avg:144.37ms
step:183/1480 train_time:24977ms step_avg:144.38ms
step:184/1480 train_time:25124ms step_avg:144.39ms
step:185/1480 train_time:25271ms step_avg:144.41ms
step:186/1480 train_time:25418ms step_avg:144.42ms
step:187/1480 train_time:25566ms step_avg:144.44ms
step:188/1480 train_time:25712ms step_avg:144.45ms
step:189/1480 train_time:25859ms step_avg:144.46ms
step:190/1480 train_time:26006ms step_avg:144.48ms
step:191/1480 train_time:26153ms step_avg:144.49ms
step:192/1480 train_time:26299ms step_avg:144.50ms
step:193/1480 train_time:26447ms step_avg:144.52ms
step:194/1480 train_time:26594ms step_avg:144.53ms
step:195/1480 train_time:26741ms step_avg:144.54ms
step:196/1480 train_time:26888ms step_avg:144.56ms
step:197/1480 train_time:27036ms step_avg:144.58ms
step:198/1480 train_time:27183ms step_avg:144.59ms
step:199/1480 train_time:27331ms step_avg:144.61ms
step:200/1480 train_time:27477ms step_avg:144.62ms
step:201/1480 train_time:27625ms step_avg:144.63ms
step:202/1480 train_time:27771ms step_avg:144.64ms
step:203/1480 train_time:27917ms step_avg:144.65ms
step:204/1480 train_time:28065ms step_avg:144.66ms
step:205/1480 train_time:28211ms step_avg:144.67ms
step:206/1480 train_time:28357ms step_avg:144.68ms
step:207/1480 train_time:28504ms step_avg:144.69ms
step:208/1480 train_time:28650ms step_avg:144.70ms
step:209/1480 train_time:28796ms step_avg:144.70ms
step:210/1480 train_time:28945ms step_avg:144.73ms
step:211/1480 train_time:29091ms step_avg:144.73ms
step:212/1480 train_time:29237ms step_avg:144.74ms
step:213/1480 train_time:29384ms step_avg:144.75ms
step:214/1480 train_time:29532ms step_avg:144.76ms
step:215/1480 train_time:29678ms step_avg:144.77ms
step:216/1480 train_time:29826ms step_avg:144.79ms
step:217/1480 train_time:29973ms step_avg:144.80ms
step:218/1480 train_time:30120ms step_avg:144.81ms
step:219/1480 train_time:30266ms step_avg:144.82ms
step:220/1480 train_time:30414ms step_avg:144.83ms
step:221/1480 train_time:30563ms step_avg:144.85ms
step:222/1480 train_time:30713ms step_avg:144.87ms
step:223/1480 train_time:30865ms step_avg:144.91ms
step:224/1480 train_time:31016ms step_avg:144.93ms
step:225/1480 train_time:31167ms step_avg:144.96ms
step:226/1480 train_time:31315ms step_avg:144.98ms
step:227/1480 train_time:31466ms step_avg:145.01ms
step:228/1480 train_time:31617ms step_avg:145.03ms
step:229/1480 train_time:31768ms step_avg:145.06ms
step:230/1480 train_time:31918ms step_avg:145.08ms
step:231/1480 train_time:32069ms step_avg:145.11ms
step:232/1480 train_time:32219ms step_avg:145.13ms
step:233/1480 train_time:32368ms step_avg:145.15ms
step:234/1480 train_time:32518ms step_avg:145.17ms
step:235/1480 train_time:32669ms step_avg:145.20ms
step:236/1480 train_time:32819ms step_avg:145.22ms
step:237/1480 train_time:32969ms step_avg:145.24ms
step:238/1480 train_time:33119ms step_avg:145.26ms
step:239/1480 train_time:33270ms step_avg:145.29ms
step:240/1480 train_time:33421ms step_avg:145.31ms
step:241/1480 train_time:33571ms step_avg:145.33ms
step:242/1480 train_time:33721ms step_avg:145.35ms
step:243/1480 train_time:33871ms step_avg:145.37ms
step:244/1480 train_time:34020ms step_avg:145.38ms
step:245/1480 train_time:34171ms step_avg:145.41ms
step:246/1480 train_time:34321ms step_avg:145.43ms
step:247/1480 train_time:34472ms step_avg:145.45ms
step:248/1480 train_time:34623ms step_avg:145.47ms
step:249/1480 train_time:34773ms step_avg:145.49ms
step:250/1480 train_time:34923ms step_avg:145.51ms
step:250/1480 val_loss:3.9822 train_time:34982ms step_avg:145.76ms
step:251/1480 train_time:35077ms step_avg:145.55ms
step:252/1480 train_time:35229ms step_avg:145.57ms
step:253/1480 train_time:35379ms step_avg:145.59ms
step:254/1480 train_time:35529ms step_avg:145.61ms
step:255/1480 train_time:35679ms step_avg:145.63ms
step:256/1480 train_time:35829ms step_avg:145.65ms
step:257/1480 train_time:35979ms step_avg:145.66ms
step:258/1480 train_time:36131ms step_avg:145.69ms
step:259/1480 train_time:36283ms step_avg:145.71ms
step:260/1480 train_time:36434ms step_avg:145.74ms
step:261/1480 train_time:36584ms step_avg:145.75ms
step:262/1480 train_time:36734ms step_avg:145.77ms
step:263/1480 train_time:36884ms step_avg:145.79ms
step:264/1480 train_time:37034ms step_avg:145.80ms
step:265/1480 train_time:37186ms step_avg:145.83ms
step:266/1480 train_time:37337ms step_avg:145.85ms
step:267/1480 train_time:37488ms step_avg:145.87ms
step:268/1480 train_time:37639ms step_avg:145.89ms
step:269/1480 train_time:37788ms step_avg:145.90ms
step:270/1480 train_time:37937ms step_avg:145.91ms
step:271/1480 train_time:38087ms step_avg:145.93ms
step:272/1480 train_time:38237ms step_avg:145.94ms
step:273/1480 train_time:38387ms step_avg:145.96ms
step:274/1480 train_time:38537ms step_avg:145.97ms
step:275/1480 train_time:38688ms step_avg:145.99ms
step:276/1480 train_time:38838ms step_avg:146.01ms
step:277/1480 train_time:38989ms step_avg:146.02ms
step:278/1480 train_time:39138ms step_avg:146.04ms
step:279/1480 train_time:39289ms step_avg:146.06ms
step:280/1480 train_time:39440ms step_avg:146.07ms
step:281/1480 train_time:39591ms step_avg:146.09ms
step:282/1480 train_time:39741ms step_avg:146.11ms
step:283/1480 train_time:39892ms step_avg:146.13ms
step:284/1480 train_time:40043ms step_avg:146.14ms
step:285/1480 train_time:40193ms step_avg:146.16ms
step:286/1480 train_time:40343ms step_avg:146.17ms
step:287/1480 train_time:40493ms step_avg:146.18ms
step:288/1480 train_time:40643ms step_avg:146.20ms
step:289/1480 train_time:40793ms step_avg:146.21ms
step:290/1480 train_time:40944ms step_avg:146.23ms
step:291/1480 train_time:41094ms step_avg:146.24ms
step:292/1480 train_time:41245ms step_avg:146.26ms
step:293/1480 train_time:41395ms step_avg:146.27ms
step:294/1480 train_time:41546ms step_avg:146.29ms
step:295/1480 train_time:41696ms step_avg:146.30ms
step:296/1480 train_time:41848ms step_avg:146.32ms
step:297/1480 train_time:41998ms step_avg:146.33ms
step:298/1480 train_time:42150ms step_avg:146.35ms
step:299/1480 train_time:42300ms step_avg:146.37ms
step:300/1480 train_time:42452ms step_avg:146.38ms
step:301/1480 train_time:42601ms step_avg:146.40ms
step:302/1480 train_time:42752ms step_avg:146.41ms
step:303/1480 train_time:42902ms step_avg:146.42ms
step:304/1480 train_time:43052ms step_avg:146.44ms
step:305/1480 train_time:43203ms step_avg:146.45ms
step:306/1480 train_time:43354ms step_avg:146.46ms
step:307/1480 train_time:43505ms step_avg:146.48ms
step:308/1480 train_time:43656ms step_avg:146.50ms
step:309/1480 train_time:43806ms step_avg:146.51ms
step:310/1480 train_time:43956ms step_avg:146.52ms
step:311/1480 train_time:44106ms step_avg:146.53ms
step:312/1480 train_time:44256ms step_avg:146.54ms
step:313/1480 train_time:44406ms step_avg:146.55ms
step:314/1480 train_time:44556ms step_avg:146.57ms
step:315/1480 train_time:44706ms step_avg:146.58ms
step:316/1480 train_time:44856ms step_avg:146.59ms
step:317/1480 train_time:45006ms step_avg:146.60ms
step:318/1480 train_time:45157ms step_avg:146.61ms
step:319/1480 train_time:45307ms step_avg:146.63ms
step:320/1480 train_time:45458ms step_avg:146.64ms
step:321/1480 train_time:45609ms step_avg:146.65ms
step:322/1480 train_time:45759ms step_avg:146.66ms
step:323/1480 train_time:45910ms step_avg:146.68ms
step:324/1480 train_time:46061ms step_avg:146.69ms
step:325/1480 train_time:46212ms step_avg:146.70ms
step:326/1480 train_time:46362ms step_avg:146.72ms
step:327/1480 train_time:46513ms step_avg:146.73ms
step:328/1480 train_time:46664ms step_avg:146.74ms
step:329/1480 train_time:46814ms step_avg:146.75ms
step:330/1480 train_time:46966ms step_avg:146.77ms
step:331/1480 train_time:47119ms step_avg:146.79ms
step:332/1480 train_time:47272ms step_avg:146.81ms
step:333/1480 train_time:47425ms step_avg:146.83ms
step:334/1480 train_time:47578ms step_avg:146.85ms
step:335/1480 train_time:47731ms step_avg:146.87ms
step:336/1480 train_time:47887ms step_avg:146.89ms
step:337/1480 train_time:48041ms step_avg:146.91ms
step:338/1480 train_time:48196ms step_avg:146.94ms
step:339/1480 train_time:48349ms step_avg:146.96ms
step:340/1480 train_time:48502ms step_avg:146.98ms
step:341/1480 train_time:48655ms step_avg:146.99ms
step:342/1480 train_time:48809ms step_avg:147.01ms
step:343/1480 train_time:48965ms step_avg:147.04ms
step:344/1480 train_time:49119ms step_avg:147.06ms
step:345/1480 train_time:49273ms step_avg:147.08ms
step:346/1480 train_time:49427ms step_avg:147.10ms
step:347/1480 train_time:49582ms step_avg:147.13ms
step:348/1480 train_time:49734ms step_avg:147.14ms
step:349/1480 train_time:49890ms step_avg:147.17ms
step:350/1480 train_time:50045ms step_avg:147.19ms
step:351/1480 train_time:50198ms step_avg:147.21ms
step:352/1480 train_time:50351ms step_avg:147.23ms
step:353/1480 train_time:50506ms step_avg:147.25ms
step:354/1480 train_time:50658ms step_avg:147.26ms
step:355/1480 train_time:50811ms step_avg:147.28ms
step:356/1480 train_time:50966ms step_avg:147.30ms
step:357/1480 train_time:51119ms step_avg:147.32ms
step:358/1480 train_time:51273ms step_avg:147.34ms
step:359/1480 train_time:51428ms step_avg:147.36ms
step:360/1480 train_time:51583ms step_avg:147.38ms
step:361/1480 train_time:51737ms step_avg:147.40ms
step:362/1480 train_time:51891ms step_avg:147.42ms
step:363/1480 train_time:52044ms step_avg:147.43ms
step:364/1480 train_time:52198ms step_avg:147.45ms
step:365/1480 train_time:52352ms step_avg:147.47ms
step:366/1480 train_time:52505ms step_avg:147.49ms
step:367/1480 train_time:52659ms step_avg:147.51ms
step:368/1480 train_time:52813ms step_avg:147.52ms
step:369/1480 train_time:52968ms step_avg:147.54ms
step:370/1480 train_time:53120ms step_avg:147.56ms
step:371/1480 train_time:53274ms step_avg:147.57ms
step:372/1480 train_time:53427ms step_avg:147.59ms
step:373/1480 train_time:53580ms step_avg:147.60ms
step:374/1480 train_time:53733ms step_avg:147.62ms
step:375/1480 train_time:53888ms step_avg:147.64ms
step:375/1480 val_loss:3.8027 train_time:53949ms step_avg:147.80ms
step:376/1480 train_time:54047ms step_avg:147.67ms
step:377/1480 train_time:54202ms step_avg:147.69ms
step:378/1480 train_time:54355ms step_avg:147.70ms
step:379/1480 train_time:54508ms step_avg:147.72ms
step:380/1480 train_time:54661ms step_avg:147.73ms
step:381/1480 train_time:54813ms step_avg:147.74ms
step:382/1480 train_time:54967ms step_avg:147.76ms
step:383/1480 train_time:55121ms step_avg:147.78ms
step:384/1480 train_time:55275ms step_avg:147.79ms
step:385/1480 train_time:55430ms step_avg:147.81ms
step:386/1480 train_time:55585ms step_avg:147.83ms
step:387/1480 train_time:55738ms step_avg:147.85ms
step:388/1480 train_time:55892ms step_avg:147.86ms
step:389/1480 train_time:56046ms step_avg:147.88ms
step:390/1480 train_time:56201ms step_avg:147.90ms
step:391/1480 train_time:56355ms step_avg:147.91ms
step:392/1480 train_time:56508ms step_avg:147.93ms
step:393/1480 train_time:56662ms step_avg:147.94ms
step:394/1480 train_time:56815ms step_avg:147.96ms
step:395/1480 train_time:56970ms step_avg:147.97ms
step:396/1480 train_time:57123ms step_avg:147.99ms
step:397/1480 train_time:57277ms step_avg:148.00ms
step:398/1480 train_time:57430ms step_avg:148.02ms
step:399/1480 train_time:57584ms step_avg:148.03ms
step:400/1480 train_time:57737ms step_avg:148.04ms
step:401/1480 train_time:57890ms step_avg:148.06ms
step:402/1480 train_time:58044ms step_avg:148.07ms
step:403/1480 train_time:58198ms step_avg:148.09ms
step:404/1480 train_time:58351ms step_avg:148.10ms
step:405/1480 train_time:58507ms step_avg:148.12ms
step:406/1480 train_time:58658ms step_avg:148.13ms
step:407/1480 train_time:58812ms step_avg:148.14ms
step:408/1480 train_time:58966ms step_avg:148.15ms
step:409/1480 train_time:59118ms step_avg:148.17ms
step:410/1480 train_time:59272ms step_avg:148.18ms
step:411/1480 train_time:59426ms step_avg:148.19ms
step:412/1480 train_time:59579ms step_avg:148.21ms
step:413/1480 train_time:59732ms step_avg:148.22ms
step:414/1480 train_time:59887ms step_avg:148.24ms
step:415/1480 train_time:60040ms step_avg:148.25ms
step:416/1480 train_time:60195ms step_avg:148.26ms
step:417/1480 train_time:60349ms step_avg:148.28ms
step:418/1480 train_time:60506ms step_avg:148.30ms
step:419/1480 train_time:60657ms step_avg:148.31ms
step:420/1480 train_time:60812ms step_avg:148.32ms
step:421/1480 train_time:60966ms step_avg:148.34ms
step:422/1480 train_time:61119ms step_avg:148.35ms
step:423/1480 train_time:61272ms step_avg:148.36ms
step:424/1480 train_time:61427ms step_avg:148.37ms
step:425/1480 train_time:61579ms step_avg:148.38ms
step:426/1480 train_time:61733ms step_avg:148.40ms
step:427/1480 train_time:61889ms step_avg:148.42ms
step:428/1480 train_time:62042ms step_avg:148.43ms
step:429/1480 train_time:62195ms step_avg:148.44ms
step:430/1480 train_time:62349ms step_avg:148.45ms
step:431/1480 train_time:62502ms step_avg:148.46ms
step:432/1480 train_time:62657ms step_avg:148.48ms
step:433/1480 train_time:62812ms step_avg:148.49ms
step:434/1480 train_time:62966ms step_avg:148.50ms
step:435/1480 train_time:63119ms step_avg:148.51ms
step:436/1480 train_time:63272ms step_avg:148.53ms
step:437/1480 train_time:63426ms step_avg:148.54ms
step:438/1480 train_time:63580ms step_avg:148.55ms
step:439/1480 train_time:63733ms step_avg:148.56ms
step:440/1480 train_time:63891ms step_avg:148.58ms
step:441/1480 train_time:64049ms step_avg:148.60ms
step:442/1480 train_time:64206ms step_avg:148.62ms
step:443/1480 train_time:64362ms step_avg:148.64ms
step:444/1480 train_time:64518ms step_avg:148.66ms
step:445/1480 train_time:64674ms step_avg:148.68ms
step:446/1480 train_time:64830ms step_avg:148.69ms
step:447/1480 train_time:64986ms step_avg:148.71ms
step:448/1480 train_time:65141ms step_avg:148.72ms
step:449/1480 train_time:65300ms step_avg:148.75ms
step:450/1480 train_time:65456ms step_avg:148.76ms
step:451/1480 train_time:65613ms step_avg:148.78ms
step:452/1480 train_time:65771ms step_avg:148.80ms
step:453/1480 train_time:65928ms step_avg:148.82ms
step:454/1480 train_time:66085ms step_avg:148.84ms
step:455/1480 train_time:66242ms step_avg:148.86ms
step:456/1480 train_time:66399ms step_avg:148.88ms
step:457/1480 train_time:66555ms step_avg:148.89ms
step:458/1480 train_time:66711ms step_avg:148.91ms
step:459/1480 train_time:66868ms step_avg:148.93ms
step:460/1480 train_time:67024ms step_avg:148.94ms
step:461/1480 train_time:67181ms step_avg:148.96ms
step:462/1480 train_time:67337ms step_avg:148.98ms
step:463/1480 train_time:67495ms step_avg:149.00ms
step:464/1480 train_time:67653ms step_avg:149.02ms
step:465/1480 train_time:67811ms step_avg:149.03ms
step:466/1480 train_time:67967ms step_avg:149.05ms
step:467/1480 train_time:68124ms step_avg:149.07ms
step:468/1480 train_time:68280ms step_avg:149.08ms
step:469/1480 train_time:68435ms step_avg:149.10ms
step:470/1480 train_time:68592ms step_avg:149.11ms
step:471/1480 train_time:68748ms step_avg:149.13ms
step:472/1480 train_time:68904ms step_avg:149.14ms
step:473/1480 train_time:69060ms step_avg:149.16ms
step:474/1480 train_time:69216ms step_avg:149.17ms
step:475/1480 train_time:69373ms step_avg:149.19ms
step:476/1480 train_time:69530ms step_avg:149.21ms
step:477/1480 train_time:69688ms step_avg:149.22ms
step:478/1480 train_time:69844ms step_avg:149.24ms
step:479/1480 train_time:70000ms step_avg:149.25ms
step:480/1480 train_time:70158ms step_avg:149.27ms
step:481/1480 train_time:70315ms step_avg:149.29ms
step:482/1480 train_time:70472ms step_avg:149.31ms
step:483/1480 train_time:70629ms step_avg:149.32ms
step:484/1480 train_time:70784ms step_avg:149.33ms
step:485/1480 train_time:70941ms step_avg:149.35ms
step:486/1480 train_time:71098ms step_avg:149.37ms
step:487/1480 train_time:71256ms step_avg:149.38ms
step:488/1480 train_time:71414ms step_avg:149.40ms
step:489/1480 train_time:71571ms step_avg:149.42ms
step:490/1480 train_time:71727ms step_avg:149.43ms
step:491/1480 train_time:71884ms step_avg:149.45ms
step:492/1480 train_time:72039ms step_avg:149.46ms
step:493/1480 train_time:72197ms step_avg:149.48ms
step:494/1480 train_time:72355ms step_avg:149.49ms
step:495/1480 train_time:72514ms step_avg:149.51ms
step:496/1480 train_time:72672ms step_avg:149.53ms
step:497/1480 train_time:72831ms step_avg:149.55ms
step:498/1480 train_time:72989ms step_avg:149.57ms
step:499/1480 train_time:73149ms step_avg:149.59ms
step:500/1480 train_time:73307ms step_avg:149.61ms
step:500/1480 val_loss:3.6806 train_time:73368ms step_avg:149.73ms
step:501/1480 train_time:73465ms step_avg:149.62ms
step:502/1480 train_time:73622ms step_avg:149.64ms
step:503/1480 train_time:73780ms step_avg:149.65ms
step:504/1480 train_time:73935ms step_avg:149.67ms
step:505/1480 train_time:74089ms step_avg:149.67ms
step:506/1480 train_time:74244ms step_avg:149.69ms
step:507/1480 train_time:74402ms step_avg:149.70ms
step:508/1480 train_time:74562ms step_avg:149.72ms
step:509/1480 train_time:74719ms step_avg:149.74ms
step:510/1480 train_time:74878ms step_avg:149.76ms
step:511/1480 train_time:75036ms step_avg:149.77ms
step:512/1480 train_time:75192ms step_avg:149.78ms
step:513/1480 train_time:75347ms step_avg:149.80ms
step:514/1480 train_time:75503ms step_avg:149.81ms
step:515/1480 train_time:75659ms step_avg:149.82ms
step:516/1480 train_time:75817ms step_avg:149.84ms
step:517/1480 train_time:75975ms step_avg:149.85ms
step:518/1480 train_time:76131ms step_avg:149.86ms
step:519/1480 train_time:76287ms step_avg:149.88ms
step:520/1480 train_time:76445ms step_avg:149.89ms
step:521/1480 train_time:76602ms step_avg:149.91ms
step:522/1480 train_time:76761ms step_avg:149.92ms
step:523/1480 train_time:76919ms step_avg:149.94ms
step:524/1480 train_time:77076ms step_avg:149.95ms
step:525/1480 train_time:77232ms step_avg:149.97ms
step:526/1480 train_time:77390ms step_avg:149.98ms
step:527/1480 train_time:77545ms step_avg:149.99ms
step:528/1480 train_time:77702ms step_avg:150.00ms
step:529/1480 train_time:77860ms step_avg:150.02ms
step:530/1480 train_time:78017ms step_avg:150.03ms
step:531/1480 train_time:78174ms step_avg:150.05ms
step:532/1480 train_time:78330ms step_avg:150.06ms
step:533/1480 train_time:78486ms step_avg:150.07ms
step:534/1480 train_time:78643ms step_avg:150.08ms
step:535/1480 train_time:78800ms step_avg:150.10ms
step:536/1480 train_time:78959ms step_avg:150.11ms
step:537/1480 train_time:79117ms step_avg:150.13ms
step:538/1480 train_time:79275ms step_avg:150.14ms
step:539/1480 train_time:79433ms step_avg:150.16ms
step:540/1480 train_time:79588ms step_avg:150.17ms
step:541/1480 train_time:79745ms step_avg:150.18ms
step:542/1480 train_time:79902ms step_avg:150.19ms
step:543/1480 train_time:80059ms step_avg:150.20ms
step:544/1480 train_time:80216ms step_avg:150.22ms
step:545/1480 train_time:80372ms step_avg:150.23ms
step:546/1480 train_time:80528ms step_avg:150.24ms
step:547/1480 train_time:80686ms step_avg:150.25ms
step:548/1480 train_time:80843ms step_avg:150.27ms
step:549/1480 train_time:81000ms step_avg:150.28ms
step:550/1480 train_time:81159ms step_avg:150.29ms
step:551/1480 train_time:81317ms step_avg:150.31ms
step:552/1480 train_time:81477ms step_avg:150.33ms
step:553/1480 train_time:81640ms step_avg:150.35ms
step:554/1480 train_time:81800ms step_avg:150.37ms
step:555/1480 train_time:81962ms step_avg:150.39ms
step:556/1480 train_time:82121ms step_avg:150.41ms
step:557/1480 train_time:82282ms step_avg:150.42ms
step:558/1480 train_time:82442ms step_avg:150.44ms
step:559/1480 train_time:82601ms step_avg:150.46ms
step:560/1480 train_time:82763ms step_avg:150.48ms
step:561/1480 train_time:82921ms step_avg:150.49ms
step:562/1480 train_time:83082ms step_avg:150.51ms
step:563/1480 train_time:83240ms step_avg:150.53ms
step:564/1480 train_time:83400ms step_avg:150.54ms
step:565/1480 train_time:83559ms step_avg:150.56ms
step:566/1480 train_time:83720ms step_avg:150.57ms
step:567/1480 train_time:83880ms step_avg:150.59ms
step:568/1480 train_time:84040ms step_avg:150.61ms
step:569/1480 train_time:84199ms step_avg:150.62ms
step:570/1480 train_time:84358ms step_avg:150.64ms
step:571/1480 train_time:84517ms step_avg:150.65ms
step:572/1480 train_time:84676ms step_avg:150.67ms
step:573/1480 train_time:84837ms step_avg:150.69ms
step:574/1480 train_time:84997ms step_avg:150.70ms
step:575/1480 train_time:85159ms step_avg:150.72ms
step:576/1480 train_time:85319ms step_avg:150.74ms
step:577/1480 train_time:85479ms step_avg:150.76ms
step:578/1480 train_time:85638ms step_avg:150.77ms
step:579/1480 train_time:85797ms step_avg:150.79ms
step:580/1480 train_time:85956ms step_avg:150.80ms
step:581/1480 train_time:86117ms step_avg:150.82ms
step:582/1480 train_time:86279ms step_avg:150.84ms
step:583/1480 train_time:86439ms step_avg:150.85ms
step:584/1480 train_time:86599ms step_avg:150.87ms
step:585/1480 train_time:86757ms step_avg:150.88ms
step:586/1480 train_time:86915ms step_avg:150.89ms
step:587/1480 train_time:87074ms step_avg:150.91ms
step:588/1480 train_time:87232ms step_avg:150.92ms
step:589/1480 train_time:87391ms step_avg:150.93ms
step:590/1480 train_time:87551ms step_avg:150.95ms
step:591/1480 train_time:87709ms step_avg:150.96ms
step:592/1480 train_time:87868ms step_avg:150.98ms
step:593/1480 train_time:88028ms step_avg:150.99ms
step:594/1480 train_time:88188ms step_avg:151.01ms
step:595/1480 train_time:88348ms step_avg:151.02ms
step:596/1480 train_time:88508ms step_avg:151.04ms
step:597/1480 train_time:88667ms step_avg:151.05ms
step:598/1480 train_time:88825ms step_avg:151.06ms
step:599/1480 train_time:88983ms step_avg:151.07ms
step:600/1480 train_time:89143ms step_avg:151.09ms
step:601/1480 train_time:89302ms step_avg:151.10ms
step:602/1480 train_time:89461ms step_avg:151.12ms
step:603/1480 train_time:89622ms step_avg:151.13ms
step:604/1480 train_time:89783ms step_avg:151.15ms
step:605/1480 train_time:89943ms step_avg:151.17ms
step:606/1480 train_time:90105ms step_avg:151.18ms
step:607/1480 train_time:90266ms step_avg:151.20ms
step:608/1480 train_time:90424ms step_avg:151.21ms
step:609/1480 train_time:90583ms step_avg:151.22ms
step:610/1480 train_time:90742ms step_avg:151.24ms
step:611/1480 train_time:90903ms step_avg:151.25ms
step:612/1480 train_time:91064ms step_avg:151.27ms
step:613/1480 train_time:91224ms step_avg:151.28ms
step:614/1480 train_time:91384ms step_avg:151.30ms
step:615/1480 train_time:91542ms step_avg:151.31ms
step:616/1480 train_time:91701ms step_avg:151.32ms
step:617/1480 train_time:91862ms step_avg:151.34ms
step:618/1480 train_time:92021ms step_avg:151.35ms
step:619/1480 train_time:92182ms step_avg:151.37ms
step:620/1480 train_time:92342ms step_avg:151.38ms
step:621/1480 train_time:92502ms step_avg:151.39ms
step:622/1480 train_time:92663ms step_avg:151.41ms
step:623/1480 train_time:92824ms step_avg:151.42ms
step:624/1480 train_time:92983ms step_avg:151.44ms
step:625/1480 train_time:93143ms step_avg:151.45ms
step:625/1480 val_loss:3.6035 train_time:93208ms step_avg:151.56ms
step:626/1480 train_time:93306ms step_avg:151.47ms
step:627/1480 train_time:93464ms step_avg:151.48ms
step:628/1480 train_time:93622ms step_avg:151.49ms
step:629/1480 train_time:93781ms step_avg:151.50ms
step:630/1480 train_time:93939ms step_avg:151.52ms
step:631/1480 train_time:94096ms step_avg:151.52ms
step:632/1480 train_time:94256ms step_avg:151.54ms
step:633/1480 train_time:94416ms step_avg:151.55ms
step:634/1480 train_time:94577ms step_avg:151.57ms
step:635/1480 train_time:94738ms step_avg:151.58ms
step:636/1480 train_time:94897ms step_avg:151.59ms
step:637/1480 train_time:95059ms step_avg:151.61ms
step:638/1480 train_time:95218ms step_avg:151.62ms
step:639/1480 train_time:95376ms step_avg:151.63ms
step:640/1480 train_time:95537ms step_avg:151.65ms
step:641/1480 train_time:95696ms step_avg:151.66ms
step:642/1480 train_time:95857ms step_avg:151.67ms
step:643/1480 train_time:96018ms step_avg:151.69ms
step:644/1480 train_time:96177ms step_avg:151.70ms
step:645/1480 train_time:96336ms step_avg:151.71ms
step:646/1480 train_time:96496ms step_avg:151.72ms
step:647/1480 train_time:96656ms step_avg:151.74ms
step:648/1480 train_time:96817ms step_avg:151.75ms
step:649/1480 train_time:96977ms step_avg:151.76ms
step:650/1480 train_time:97138ms step_avg:151.78ms
step:651/1480 train_time:97297ms step_avg:151.79ms
step:652/1480 train_time:97457ms step_avg:151.80ms
step:653/1480 train_time:97617ms step_avg:151.82ms
step:654/1480 train_time:97778ms step_avg:151.83ms
step:655/1480 train_time:97937ms step_avg:151.84ms
step:656/1480 train_time:98097ms step_avg:151.85ms
step:657/1480 train_time:98258ms step_avg:151.87ms
step:658/1480 train_time:98417ms step_avg:151.88ms
step:659/1480 train_time:98580ms step_avg:151.90ms
step:660/1480 train_time:98743ms step_avg:151.91ms
step:661/1480 train_time:98905ms step_avg:151.93ms
step:662/1480 train_time:99063ms step_avg:151.94ms
step:663/1480 train_time:99222ms step_avg:151.95ms
step:664/1480 train_time:99384ms step_avg:151.96ms
step:665/1480 train_time:99546ms step_avg:151.98ms
step:666/1480 train_time:99707ms step_avg:151.99ms
step:667/1480 train_time:99867ms step_avg:152.00ms
step:668/1480 train_time:100029ms step_avg:152.02ms
step:669/1480 train_time:100191ms step_avg:152.04ms
step:670/1480 train_time:100351ms step_avg:152.05ms
step:671/1480 train_time:100513ms step_avg:152.06ms
step:672/1480 train_time:100674ms step_avg:152.08ms
step:673/1480 train_time:100838ms step_avg:152.09ms
step:674/1480 train_time:101001ms step_avg:152.11ms
step:675/1480 train_time:101162ms step_avg:152.12ms
step:676/1480 train_time:101324ms step_avg:152.14ms
step:677/1480 train_time:101484ms step_avg:152.15ms
step:678/1480 train_time:101644ms step_avg:152.16ms
step:679/1480 train_time:101804ms step_avg:152.17ms
step:680/1480 train_time:101965ms step_avg:152.19ms
step:681/1480 train_time:102126ms step_avg:152.20ms
step:682/1480 train_time:102289ms step_avg:152.22ms
step:683/1480 train_time:102451ms step_avg:152.23ms
step:684/1480 train_time:102612ms step_avg:152.24ms
step:685/1480 train_time:102777ms step_avg:152.26ms
step:686/1480 train_time:102940ms step_avg:152.28ms
step:687/1480 train_time:103100ms step_avg:152.29ms
step:688/1480 train_time:103264ms step_avg:152.31ms
step:689/1480 train_time:103427ms step_avg:152.32ms
step:690/1480 train_time:103592ms step_avg:152.34ms
step:691/1480 train_time:103753ms step_avg:152.35ms
step:692/1480 train_time:103915ms step_avg:152.37ms
step:693/1480 train_time:104077ms step_avg:152.38ms
step:694/1480 train_time:104239ms step_avg:152.40ms
step:695/1480 train_time:104400ms step_avg:152.41ms
step:696/1480 train_time:104561ms step_avg:152.42ms
step:697/1480 train_time:104722ms step_avg:152.43ms
step:698/1480 train_time:104883ms step_avg:152.45ms
step:699/1480 train_time:105046ms step_avg:152.46ms
step:700/1480 train_time:105207ms step_avg:152.47ms
step:701/1480 train_time:105366ms step_avg:152.48ms
step:702/1480 train_time:105527ms step_avg:152.50ms
step:703/1480 train_time:105689ms step_avg:152.51ms
step:704/1480 train_time:105850ms step_avg:152.52ms
step:705/1480 train_time:106013ms step_avg:152.54ms
step:706/1480 train_time:106176ms step_avg:152.55ms
step:707/1480 train_time:106338ms step_avg:152.57ms
step:708/1480 train_time:106498ms step_avg:152.58ms
step:709/1480 train_time:106661ms step_avg:152.59ms
step:710/1480 train_time:106822ms step_avg:152.60ms
step:711/1480 train_time:106983ms step_avg:152.61ms
step:712/1480 train_time:107147ms step_avg:152.63ms
step:713/1480 train_time:107311ms step_avg:152.65ms
step:714/1480 train_time:107473ms step_avg:152.66ms
step:715/1480 train_time:107635ms step_avg:152.67ms
step:716/1480 train_time:107796ms step_avg:152.68ms
step:717/1480 train_time:107959ms step_avg:152.70ms
step:718/1480 train_time:108118ms step_avg:152.71ms
step:719/1480 train_time:108279ms step_avg:152.72ms
step:720/1480 train_time:108441ms step_avg:152.73ms
step:721/1480 train_time:108602ms step_avg:152.75ms
step:722/1480 train_time:108764ms step_avg:152.76ms
step:723/1480 train_time:108925ms step_avg:152.77ms
step:724/1480 train_time:109087ms step_avg:152.78ms
step:725/1480 train_time:109250ms step_avg:152.80ms
step:726/1480 train_time:109414ms step_avg:152.81ms
step:727/1480 train_time:109576ms step_avg:152.83ms
step:728/1480 train_time:109737ms step_avg:152.84ms
step:729/1480 train_time:109898ms step_avg:152.85ms
step:730/1480 train_time:110061ms step_avg:152.86ms
step:731/1480 train_time:110222ms step_avg:152.87ms
step:732/1480 train_time:110381ms step_avg:152.88ms
step:733/1480 train_time:110544ms step_avg:152.90ms
step:734/1480 train_time:110704ms step_avg:152.91ms
step:735/1480 train_time:110864ms step_avg:152.92ms
step:736/1480 train_time:111027ms step_avg:152.93ms
step:737/1480 train_time:111187ms step_avg:152.94ms
step:738/1480 train_time:111349ms step_avg:152.95ms
step:739/1480 train_time:111509ms step_avg:152.96ms
step:740/1480 train_time:111675ms step_avg:152.98ms
step:741/1480 train_time:111838ms step_avg:152.99ms
step:742/1480 train_time:111999ms step_avg:153.00ms
step:743/1480 train_time:112161ms step_avg:153.02ms
step:744/1480 train_time:112323ms step_avg:153.03ms
step:745/1480 train_time:112488ms step_avg:153.05ms
step:746/1480 train_time:112648ms step_avg:153.05ms
step:747/1480 train_time:112811ms step_avg:153.07ms
step:748/1480 train_time:112977ms step_avg:153.08ms
step:749/1480 train_time:113142ms step_avg:153.10ms
step:750/1480 train_time:113301ms step_avg:153.11ms
step:750/1480 val_loss:3.5468 train_time:113366ms step_avg:153.20ms
step:751/1480 train_time:113466ms step_avg:153.13ms
step:752/1480 train_time:113628ms step_avg:153.14ms
step:753/1480 train_time:113789ms step_avg:153.15ms
step:754/1480 train_time:113949ms step_avg:153.16ms
step:755/1480 train_time:114109ms step_avg:153.17ms
step:756/1480 train_time:114269ms step_avg:153.18ms
step:757/1480 train_time:114432ms step_avg:153.19ms
step:758/1480 train_time:114593ms step_avg:153.20ms
step:759/1480 train_time:114755ms step_avg:153.21ms
step:760/1480 train_time:114916ms step_avg:153.22ms
step:761/1480 train_time:115080ms step_avg:153.24ms
step:762/1480 train_time:115242ms step_avg:153.25ms
step:763/1480 train_time:115405ms step_avg:153.26ms
step:764/1480 train_time:115566ms step_avg:153.27ms
step:765/1480 train_time:115728ms step_avg:153.28ms
step:766/1480 train_time:115891ms step_avg:153.30ms
step:767/1480 train_time:116053ms step_avg:153.31ms
step:768/1480 train_time:116215ms step_avg:153.32ms
step:769/1480 train_time:116379ms step_avg:153.33ms
step:770/1480 train_time:116543ms step_avg:153.35ms
step:771/1480 train_time:116707ms step_avg:153.36ms
step:772/1480 train_time:116868ms step_avg:153.37ms
step:773/1480 train_time:117030ms step_avg:153.38ms
step:774/1480 train_time:117192ms step_avg:153.39ms
step:775/1480 train_time:117355ms step_avg:153.40ms
step:776/1480 train_time:117520ms step_avg:153.42ms
step:777/1480 train_time:117687ms step_avg:153.44ms
step:778/1480 train_time:117849ms step_avg:153.45ms
step:779/1480 train_time:118010ms step_avg:153.46ms
step:780/1480 train_time:118173ms step_avg:153.47ms
step:781/1480 train_time:118336ms step_avg:153.48ms
step:782/1480 train_time:118501ms step_avg:153.50ms
step:783/1480 train_time:118663ms step_avg:153.51ms
step:784/1480 train_time:118827ms step_avg:153.52ms
step:785/1480 train_time:118989ms step_avg:153.53ms
step:786/1480 train_time:119156ms step_avg:153.55ms
step:787/1480 train_time:119320ms step_avg:153.56ms
step:788/1480 train_time:119485ms step_avg:153.58ms
step:789/1480 train_time:119647ms step_avg:153.59ms
step:790/1480 train_time:119811ms step_avg:153.60ms
step:791/1480 train_time:119978ms step_avg:153.62ms
step:792/1480 train_time:120145ms step_avg:153.64ms
step:793/1480 train_time:120306ms step_avg:153.65ms
step:794/1480 train_time:120469ms step_avg:153.66ms
step:795/1480 train_time:120635ms step_avg:153.67ms
step:796/1480 train_time:120803ms step_avg:153.69ms
step:797/1480 train_time:120967ms step_avg:153.71ms
step:798/1480 train_time:121130ms step_avg:153.72ms
step:799/1480 train_time:121298ms step_avg:153.74ms
step:800/1480 train_time:121461ms step_avg:153.75ms
step:801/1480 train_time:121625ms step_avg:153.76ms
step:802/1480 train_time:121792ms step_avg:153.78ms
step:803/1480 train_time:121954ms step_avg:153.79ms
step:804/1480 train_time:122117ms step_avg:153.80ms
step:805/1480 train_time:122282ms step_avg:153.81ms
step:806/1480 train_time:122445ms step_avg:153.83ms
step:807/1480 train_time:122606ms step_avg:153.83ms
step:808/1480 train_time:122769ms step_avg:153.85ms
step:809/1480 train_time:122932ms step_avg:153.86ms
step:810/1480 train_time:123094ms step_avg:153.87ms
step:811/1480 train_time:123256ms step_avg:153.88ms
step:812/1480 train_time:123421ms step_avg:153.89ms
step:813/1480 train_time:123582ms step_avg:153.90ms
step:814/1480 train_time:123746ms step_avg:153.91ms
step:815/1480 train_time:123908ms step_avg:153.92ms
step:816/1480 train_time:124073ms step_avg:153.94ms
step:817/1480 train_time:124234ms step_avg:153.94ms
step:818/1480 train_time:124396ms step_avg:153.96ms
step:819/1480 train_time:124560ms step_avg:153.97ms
step:820/1480 train_time:124724ms step_avg:153.98ms
step:821/1480 train_time:124885ms step_avg:153.99ms
step:822/1480 train_time:125049ms step_avg:154.00ms
step:823/1480 train_time:125210ms step_avg:154.01ms
step:824/1480 train_time:125372ms step_avg:154.02ms
step:825/1480 train_time:125538ms step_avg:154.03ms
step:826/1480 train_time:125705ms step_avg:154.05ms
step:827/1480 train_time:125869ms step_avg:154.06ms
step:828/1480 train_time:126031ms step_avg:154.07ms
step:829/1480 train_time:126194ms step_avg:154.08ms
step:830/1480 train_time:126359ms step_avg:154.10ms
step:831/1480 train_time:126524ms step_avg:154.11ms
step:832/1480 train_time:126688ms step_avg:154.12ms
step:833/1480 train_time:126851ms step_avg:154.13ms
step:834/1480 train_time:127017ms step_avg:154.15ms
step:835/1480 train_time:127180ms step_avg:154.16ms
step:836/1480 train_time:127344ms step_avg:154.17ms
step:837/1480 train_time:127506ms step_avg:154.18ms
step:838/1480 train_time:127668ms step_avg:154.19ms
step:839/1480 train_time:127829ms step_avg:154.20ms
step:840/1480 train_time:127989ms step_avg:154.20ms
step:841/1480 train_time:128150ms step_avg:154.21ms
step:842/1480 train_time:128313ms step_avg:154.22ms
step:843/1480 train_time:128476ms step_avg:154.23ms
step:844/1480 train_time:128637ms step_avg:154.24ms
step:845/1480 train_time:128803ms step_avg:154.25ms
step:846/1480 train_time:128967ms step_avg:154.27ms
step:847/1480 train_time:129131ms step_avg:154.28ms
step:848/1480 train_time:129292ms step_avg:154.29ms
step:849/1480 train_time:129456ms step_avg:154.30ms
step:850/1480 train_time:129618ms step_avg:154.31ms
step:851/1480 train_time:129783ms step_avg:154.32ms
step:852/1480 train_time:129946ms step_avg:154.33ms
step:853/1480 train_time:130108ms step_avg:154.34ms
step:854/1480 train_time:130272ms step_avg:154.35ms
step:855/1480 train_time:130436ms step_avg:154.36ms
step:856/1480 train_time:130599ms step_avg:154.37ms
step:857/1480 train_time:130764ms step_avg:154.38ms
step:858/1480 train_time:130929ms step_avg:154.40ms
step:859/1480 train_time:131093ms step_avg:154.41ms
step:860/1480 train_time:131254ms step_avg:154.42ms
step:861/1480 train_time:131421ms step_avg:154.43ms
step:862/1480 train_time:131590ms step_avg:154.45ms
step:863/1480 train_time:131757ms step_avg:154.46ms
step:864/1480 train_time:131922ms step_avg:154.48ms
step:865/1480 train_time:132084ms step_avg:154.48ms
step:866/1480 train_time:132250ms step_avg:154.50ms
step:867/1480 train_time:132413ms step_avg:154.51ms
step:868/1480 train_time:132574ms step_avg:154.52ms
step:869/1480 train_time:132736ms step_avg:154.52ms
step:870/1480 train_time:132902ms step_avg:154.54ms
step:871/1480 train_time:133065ms step_avg:154.55ms
step:872/1480 train_time:133229ms step_avg:154.56ms
step:873/1480 train_time:133391ms step_avg:154.57ms
step:874/1480 train_time:133559ms step_avg:154.58ms
step:875/1480 train_time:133725ms step_avg:154.59ms
step:875/1480 val_loss:3.5027 train_time:133790ms step_avg:154.67ms
step:876/1480 train_time:133891ms step_avg:154.61ms
step:877/1480 train_time:134058ms step_avg:154.62ms
step:878/1480 train_time:134220ms step_avg:154.63ms
step:879/1480 train_time:134384ms step_avg:154.64ms
step:880/1480 train_time:134547ms step_avg:154.65ms
step:881/1480 train_time:134709ms step_avg:154.66ms
step:882/1480 train_time:134875ms step_avg:154.67ms
step:883/1480 train_time:135041ms step_avg:154.69ms
step:884/1480 train_time:135207ms step_avg:154.70ms
step:885/1480 train_time:135372ms step_avg:154.71ms
step:886/1480 train_time:135539ms step_avg:154.72ms
step:887/1480 train_time:135707ms step_avg:154.74ms
step:888/1480 train_time:135882ms step_avg:154.76ms
step:889/1480 train_time:136050ms step_avg:154.78ms
step:890/1480 train_time:136212ms step_avg:154.79ms
step:891/1480 train_time:136379ms step_avg:154.80ms
step:892/1480 train_time:136545ms step_avg:154.81ms
step:893/1480 train_time:136707ms step_avg:154.82ms
step:894/1480 train_time:136874ms step_avg:154.84ms
step:895/1480 train_time:137042ms step_avg:154.85ms
step:896/1480 train_time:137206ms step_avg:154.86ms
step:897/1480 train_time:137372ms step_avg:154.87ms
step:898/1480 train_time:137539ms step_avg:154.89ms
step:899/1480 train_time:137702ms step_avg:154.90ms
step:900/1480 train_time:137865ms step_avg:154.90ms
step:901/1480 train_time:138029ms step_avg:154.91ms
step:902/1480 train_time:138193ms step_avg:154.92ms
step:903/1480 train_time:138364ms step_avg:154.94ms
step:904/1480 train_time:138530ms step_avg:154.96ms
step:905/1480 train_time:138692ms step_avg:154.96ms
step:906/1480 train_time:138860ms step_avg:154.98ms
step:907/1480 train_time:139028ms step_avg:154.99ms
step:908/1480 train_time:139190ms step_avg:155.00ms
step:909/1480 train_time:139356ms step_avg:155.01ms
step:910/1480 train_time:139526ms step_avg:155.03ms
step:911/1480 train_time:139691ms step_avg:155.04ms
step:912/1480 train_time:139859ms step_avg:155.05ms
step:913/1480 train_time:140026ms step_avg:155.07ms
step:914/1480 train_time:140192ms step_avg:155.08ms
step:915/1480 train_time:140363ms step_avg:155.10ms
step:916/1480 train_time:140527ms step_avg:155.11ms
step:917/1480 train_time:140691ms step_avg:155.12ms
step:918/1480 train_time:140861ms step_avg:155.13ms
step:919/1480 train_time:141030ms step_avg:155.15ms
step:920/1480 train_time:141196ms step_avg:155.16ms
step:921/1480 train_time:141360ms step_avg:155.17ms
step:922/1480 train_time:141528ms step_avg:155.18ms
step:923/1480 train_time:141690ms step_avg:155.19ms
step:924/1480 train_time:141856ms step_avg:155.20ms
step:925/1480 train_time:142021ms step_avg:155.21ms
step:926/1480 train_time:142184ms step_avg:155.22ms
step:927/1480 train_time:142348ms step_avg:155.23ms
step:928/1480 train_time:142514ms step_avg:155.24ms
step:929/1480 train_time:142679ms step_avg:155.25ms
step:930/1480 train_time:142845ms step_avg:155.27ms
step:931/1480 train_time:143009ms step_avg:155.28ms
step:932/1480 train_time:143176ms step_avg:155.29ms
step:933/1480 train_time:143343ms step_avg:155.30ms
step:934/1480 train_time:143509ms step_avg:155.31ms
step:935/1480 train_time:143680ms step_avg:155.33ms
step:936/1480 train_time:143847ms step_avg:155.34ms
step:937/1480 train_time:144018ms step_avg:155.36ms
step:938/1480 train_time:144182ms step_avg:155.37ms
step:939/1480 train_time:144352ms step_avg:155.38ms
step:940/1480 train_time:144518ms step_avg:155.40ms
step:941/1480 train_time:144682ms step_avg:155.41ms
step:942/1480 train_time:144847ms step_avg:155.42ms
step:943/1480 train_time:145019ms step_avg:155.43ms
step:944/1480 train_time:145192ms step_avg:155.45ms
step:945/1480 train_time:145355ms step_avg:155.46ms
step:946/1480 train_time:145523ms step_avg:155.47ms
step:947/1480 train_time:145689ms step_avg:155.48ms
step:948/1480 train_time:145857ms step_avg:155.50ms
step:949/1480 train_time:146022ms step_avg:155.51ms
step:950/1480 train_time:146185ms step_avg:155.52ms
step:951/1480 train_time:146354ms step_avg:155.53ms
step:952/1480 train_time:146520ms step_avg:155.54ms
step:953/1480 train_time:146689ms step_avg:155.56ms
step:954/1480 train_time:146860ms step_avg:155.57ms
step:955/1480 train_time:147023ms step_avg:155.58ms
step:956/1480 train_time:147188ms step_avg:155.59ms
step:957/1480 train_time:147358ms step_avg:155.60ms
step:958/1480 train_time:147527ms step_avg:155.62ms
step:959/1480 train_time:147691ms step_avg:155.63ms
step:960/1480 train_time:147859ms step_avg:155.64ms
step:961/1480 train_time:148024ms step_avg:155.65ms
step:962/1480 train_time:148188ms step_avg:155.66ms
step:963/1480 train_time:148355ms step_avg:155.67ms
step:964/1480 train_time:148524ms step_avg:155.69ms
step:965/1480 train_time:148687ms step_avg:155.69ms
step:966/1480 train_time:148853ms step_avg:155.70ms
step:967/1480 train_time:149017ms step_avg:155.71ms
step:968/1480 train_time:149181ms step_avg:155.72ms
step:969/1480 train_time:149347ms step_avg:155.73ms
step:970/1480 train_time:149510ms step_avg:155.74ms
step:971/1480 train_time:149676ms step_avg:155.75ms
step:972/1480 train_time:149840ms step_avg:155.76ms
step:973/1480 train_time:150004ms step_avg:155.77ms
step:974/1480 train_time:150174ms step_avg:155.78ms
step:975/1480 train_time:150341ms step_avg:155.79ms
step:976/1480 train_time:150505ms step_avg:155.80ms
step:977/1480 train_time:150669ms step_avg:155.81ms
step:978/1480 train_time:150834ms step_avg:155.82ms
step:979/1480 train_time:151001ms step_avg:155.83ms
step:980/1480 train_time:151166ms step_avg:155.84ms
step:981/1480 train_time:151336ms step_avg:155.86ms
step:982/1480 train_time:151500ms step_avg:155.86ms
step:983/1480 train_time:151665ms step_avg:155.87ms
step:984/1480 train_time:151828ms step_avg:155.88ms
step:985/1480 train_time:151997ms step_avg:155.89ms
step:986/1480 train_time:152161ms step_avg:155.90ms
step:987/1480 train_time:152324ms step_avg:155.91ms
step:988/1480 train_time:152492ms step_avg:155.92ms
step:989/1480 train_time:152659ms step_avg:155.93ms
step:990/1480 train_time:152827ms step_avg:155.95ms
step:991/1480 train_time:152994ms step_avg:155.96ms
step:992/1480 train_time:153168ms step_avg:155.98ms
step:993/1480 train_time:153345ms step_avg:156.00ms
step:994/1480 train_time:153510ms step_avg:156.01ms
step:995/1480 train_time:153676ms step_avg:156.02ms
step:996/1480 train_time:153839ms step_avg:156.02ms
step:997/1480 train_time:154002ms step_avg:156.03ms
step:998/1480 train_time:154167ms step_avg:156.04ms
step:999/1480 train_time:154334ms step_avg:156.05ms
step:1000/1480 train_time:154503ms step_avg:156.06ms
step:1000/1480 val_loss:3.4384 train_time:154573ms step_avg:156.13ms
step:1001/1480 train_time:154676ms step_avg:156.08ms
step:1002/1480 train_time:154842ms step_avg:156.09ms
step:1003/1480 train_time:155015ms step_avg:156.11ms
step:1004/1480 train_time:155183ms step_avg:156.12ms
step:1005/1480 train_time:155352ms step_avg:156.13ms
step:1006/1480 train_time:155519ms step_avg:156.14ms
step:1007/1480 train_time:155684ms step_avg:156.15ms
step:1008/1480 train_time:155852ms step_avg:156.16ms
step:1009/1480 train_time:156027ms step_avg:156.18ms
step:1010/1480 train_time:156192ms step_avg:156.19ms
step:1011/1480 train_time:156357ms step_avg:156.20ms
step:1012/1480 train_time:156522ms step_avg:156.21ms
step:1013/1480 train_time:156692ms step_avg:156.22ms
step:1014/1480 train_time:156858ms step_avg:156.23ms
step:1015/1480 train_time:157028ms step_avg:156.25ms
step:1016/1480 train_time:157198ms step_avg:156.26ms
step:1017/1480 train_time:157370ms step_avg:156.28ms
step:1018/1480 train_time:157539ms step_avg:156.29ms
step:1019/1480 train_time:157710ms step_avg:156.30ms
step:1020/1480 train_time:157879ms step_avg:156.32ms
step:1021/1480 train_time:158044ms step_avg:156.32ms
step:1022/1480 train_time:158211ms step_avg:156.34ms
step:1023/1480 train_time:158378ms step_avg:156.35ms
step:1024/1480 train_time:158543ms step_avg:156.35ms
step:1025/1480 train_time:158716ms step_avg:156.37ms
step:1026/1480 train_time:158882ms step_avg:156.38ms
step:1027/1480 train_time:159048ms step_avg:156.39ms
step:1028/1480 train_time:159221ms step_avg:156.41ms
step:1029/1480 train_time:159394ms step_avg:156.42ms
step:1030/1480 train_time:159560ms step_avg:156.43ms
step:1031/1480 train_time:159723ms step_avg:156.44ms
step:1032/1480 train_time:159897ms step_avg:156.45ms
step:1033/1480 train_time:160062ms step_avg:156.46ms
step:1034/1480 train_time:160230ms step_avg:156.47ms
step:1035/1480 train_time:160399ms step_avg:156.49ms
step:1036/1480 train_time:160564ms step_avg:156.49ms
step:1037/1480 train_time:160731ms step_avg:156.51ms
step:1038/1480 train_time:160898ms step_avg:156.52ms
step:1039/1480 train_time:161067ms step_avg:156.53ms
step:1040/1480 train_time:161234ms step_avg:156.54ms
step:1041/1480 train_time:161401ms step_avg:156.55ms
step:1042/1480 train_time:161564ms step_avg:156.55ms
step:1043/1480 train_time:161730ms step_avg:156.56ms
step:1044/1480 train_time:161897ms step_avg:156.57ms
step:1045/1480 train_time:162066ms step_avg:156.59ms
step:1046/1480 train_time:162233ms step_avg:156.60ms
step:1047/1480 train_time:162400ms step_avg:156.61ms
step:1048/1480 train_time:162566ms step_avg:156.61ms
step:1049/1480 train_time:162732ms step_avg:156.62ms
step:1050/1480 train_time:162901ms step_avg:156.64ms
step:1051/1480 train_time:163070ms step_avg:156.65ms
step:1052/1480 train_time:163239ms step_avg:156.66ms
step:1053/1480 train_time:163405ms step_avg:156.67ms
step:1054/1480 train_time:163575ms step_avg:156.68ms
step:1055/1480 train_time:163741ms step_avg:156.69ms
step:1056/1480 train_time:163905ms step_avg:156.70ms
step:1057/1480 train_time:164072ms step_avg:156.71ms
step:1058/1480 train_time:164241ms step_avg:156.72ms
step:1059/1480 train_time:164417ms step_avg:156.74ms
step:1060/1480 train_time:164586ms step_avg:156.75ms
step:1061/1480 train_time:164750ms step_avg:156.76ms
step:1062/1480 train_time:164917ms step_avg:156.76ms
step:1063/1480 train_time:165081ms step_avg:156.77ms
step:1064/1480 train_time:165247ms step_avg:156.78ms
step:1065/1480 train_time:165415ms step_avg:156.79ms
step:1066/1480 train_time:165582ms step_avg:156.80ms
step:1067/1480 train_time:165751ms step_avg:156.81ms
step:1068/1480 train_time:165918ms step_avg:156.82ms
step:1069/1480 train_time:166089ms step_avg:156.84ms
step:1070/1480 train_time:166257ms step_avg:156.85ms
step:1071/1480 train_time:166429ms step_avg:156.86ms
step:1072/1480 train_time:166596ms step_avg:156.87ms
step:1073/1480 train_time:166759ms step_avg:156.88ms
step:1074/1480 train_time:166926ms step_avg:156.89ms
step:1075/1480 train_time:167097ms step_avg:156.90ms
step:1076/1480 train_time:167265ms step_avg:156.91ms
step:1077/1480 train_time:167429ms step_avg:156.92ms
step:1078/1480 train_time:167604ms step_avg:156.93ms
step:1079/1480 train_time:167776ms step_avg:156.95ms
step:1080/1480 train_time:167946ms step_avg:156.96ms
step:1081/1480 train_time:168114ms step_avg:156.97ms
step:1082/1480 train_time:168280ms step_avg:156.98ms
step:1083/1480 train_time:168448ms step_avg:156.99ms
step:1084/1480 train_time:168616ms step_avg:157.00ms
step:1085/1480 train_time:168784ms step_avg:157.01ms
step:1086/1480 train_time:168952ms step_avg:157.02ms
step:1087/1480 train_time:169119ms step_avg:157.03ms
step:1088/1480 train_time:169288ms step_avg:157.04ms
step:1089/1480 train_time:169459ms step_avg:157.05ms
step:1090/1480 train_time:169631ms step_avg:157.07ms
step:1091/1480 train_time:169800ms step_avg:157.08ms
step:1092/1480 train_time:169967ms step_avg:157.09ms
step:1093/1480 train_time:170136ms step_avg:157.10ms
step:1094/1480 train_time:170302ms step_avg:157.10ms
step:1095/1480 train_time:170465ms step_avg:157.11ms
step:1096/1480 train_time:170634ms step_avg:157.12ms
step:1097/1480 train_time:170803ms step_avg:157.13ms
step:1098/1480 train_time:170974ms step_avg:157.15ms
step:1099/1480 train_time:171144ms step_avg:157.16ms
step:1100/1480 train_time:171317ms step_avg:157.17ms
step:1101/1480 train_time:171487ms step_avg:157.18ms
step:1102/1480 train_time:171657ms step_avg:157.20ms
step:1103/1480 train_time:171834ms step_avg:157.21ms
step:1104/1480 train_time:172002ms step_avg:157.22ms
step:1105/1480 train_time:172171ms step_avg:157.23ms
step:1106/1480 train_time:172340ms step_avg:157.24ms
step:1107/1480 train_time:172510ms step_avg:157.26ms
step:1108/1480 train_time:172676ms step_avg:157.26ms
step:1109/1480 train_time:172841ms step_avg:157.27ms
step:1110/1480 train_time:173007ms step_avg:157.28ms
step:1111/1480 train_time:173175ms step_avg:157.29ms
step:1112/1480 train_time:173345ms step_avg:157.30ms
step:1113/1480 train_time:173525ms step_avg:157.32ms
step:1114/1480 train_time:173699ms step_avg:157.34ms
step:1115/1480 train_time:173872ms step_avg:157.35ms
step:1116/1480 train_time:174039ms step_avg:157.36ms
step:1117/1480 train_time:174214ms step_avg:157.37ms
step:1118/1480 train_time:174390ms step_avg:157.39ms
step:1119/1480 train_time:174556ms step_avg:157.40ms
step:1120/1480 train_time:174724ms step_avg:157.41ms
step:1121/1480 train_time:174895ms step_avg:157.42ms
step:1122/1480 train_time:175061ms step_avg:157.43ms
step:1123/1480 train_time:175228ms step_avg:157.44ms
step:1124/1480 train_time:175398ms step_avg:157.45ms
step:1125/1480 train_time:175565ms step_avg:157.46ms
step:1125/1480 val_loss:3.3840 train_time:175633ms step_avg:157.52ms
step:1126/1480 train_time:175734ms step_avg:157.47ms
step:1127/1480 train_time:175907ms step_avg:157.48ms
step:1128/1480 train_time:176079ms step_avg:157.49ms
step:1129/1480 train_time:176251ms step_avg:157.51ms
step:1130/1480 train_time:176421ms step_avg:157.52ms
step:1131/1480 train_time:176600ms step_avg:157.54ms
step:1132/1480 train_time:176766ms step_avg:157.55ms
step:1133/1480 train_time:176939ms step_avg:157.56ms
step:1134/1480 train_time:177110ms step_avg:157.57ms
step:1135/1480 train_time:177278ms step_avg:157.58ms
step:1136/1480 train_time:177448ms step_avg:157.59ms
step:1137/1480 train_time:177616ms step_avg:157.60ms
step:1138/1480 train_time:177786ms step_avg:157.61ms
step:1139/1480 train_time:177954ms step_avg:157.62ms
step:1140/1480 train_time:178124ms step_avg:157.63ms
step:1141/1480 train_time:178294ms step_avg:157.64ms
step:1142/1480 train_time:178464ms step_avg:157.65ms
step:1143/1480 train_time:178635ms step_avg:157.67ms
step:1144/1480 train_time:178804ms step_avg:157.68ms
step:1145/1480 train_time:178970ms step_avg:157.68ms
step:1146/1480 train_time:179142ms step_avg:157.70ms
step:1147/1480 train_time:179310ms step_avg:157.70ms
step:1148/1480 train_time:179479ms step_avg:157.71ms
step:1149/1480 train_time:179651ms step_avg:157.73ms
step:1150/1480 train_time:179819ms step_avg:157.74ms
step:1151/1480 train_time:179990ms step_avg:157.75ms
step:1152/1480 train_time:180161ms step_avg:157.76ms
step:1153/1480 train_time:180333ms step_avg:157.77ms
step:1154/1480 train_time:180500ms step_avg:157.78ms
step:1155/1480 train_time:180672ms step_avg:157.79ms
step:1156/1480 train_time:180851ms step_avg:157.81ms
step:1157/1480 train_time:181021ms step_avg:157.82ms
step:1158/1480 train_time:181188ms step_avg:157.83ms
step:1159/1480 train_time:181354ms step_avg:157.84ms
step:1160/1480 train_time:181521ms step_avg:157.84ms
step:1161/1480 train_time:181691ms step_avg:157.85ms
step:1162/1480 train_time:181860ms step_avg:157.86ms
step:1163/1480 train_time:182030ms step_avg:157.88ms
step:1164/1480 train_time:182200ms step_avg:157.89ms
step:1165/1480 train_time:182365ms step_avg:157.89ms
step:1166/1480 train_time:182534ms step_avg:157.90ms
step:1167/1480 train_time:182701ms step_avg:157.91ms
step:1168/1480 train_time:182868ms step_avg:157.92ms
step:1169/1480 train_time:183037ms step_avg:157.93ms
step:1170/1480 train_time:183206ms step_avg:157.94ms
step:1171/1480 train_time:183372ms step_avg:157.94ms
step:1172/1480 train_time:183539ms step_avg:157.95ms
step:1173/1480 train_time:183710ms step_avg:157.96ms
step:1174/1480 train_time:183891ms step_avg:157.98ms
step:1175/1480 train_time:184063ms step_avg:157.99ms
step:1176/1480 train_time:184237ms step_avg:158.01ms
step:1177/1480 train_time:184413ms step_avg:158.02ms
step:1178/1480 train_time:184580ms step_avg:158.03ms
step:1179/1480 train_time:184747ms step_avg:158.04ms
step:1180/1480 train_time:184927ms step_avg:158.06ms
step:1181/1480 train_time:185098ms step_avg:158.07ms
step:1182/1480 train_time:185266ms step_avg:158.08ms
step:1183/1480 train_time:185437ms step_avg:158.09ms
step:1184/1480 train_time:185605ms step_avg:158.10ms
step:1185/1480 train_time:185777ms step_avg:158.11ms
step:1186/1480 train_time:185949ms step_avg:158.12ms
step:1187/1480 train_time:186131ms step_avg:158.14ms
step:1188/1480 train_time:186297ms step_avg:158.15ms
step:1189/1480 train_time:186468ms step_avg:158.16ms
step:1190/1480 train_time:186637ms step_avg:158.17ms
step:1191/1480 train_time:186808ms step_avg:158.18ms
step:1192/1480 train_time:186974ms step_avg:158.18ms
step:1193/1480 train_time:187142ms step_avg:158.19ms
step:1194/1480 train_time:187311ms step_avg:158.20ms
step:1195/1480 train_time:187485ms step_avg:158.22ms
step:1196/1480 train_time:187666ms step_avg:158.23ms
step:1197/1480 train_time:187836ms step_avg:158.24ms
step:1198/1480 train_time:188019ms step_avg:158.26ms
step:1199/1480 train_time:188188ms step_avg:158.27ms
step:1200/1480 train_time:188357ms step_avg:158.28ms
step:1201/1480 train_time:188526ms step_avg:158.29ms
step:1202/1480 train_time:188708ms step_avg:158.31ms
step:1203/1480 train_time:188886ms step_avg:158.33ms
step:1204/1480 train_time:189062ms step_avg:158.34ms
step:1205/1480 train_time:189230ms step_avg:158.35ms
step:1206/1480 train_time:189398ms step_avg:158.36ms
step:1207/1480 train_time:189568ms step_avg:158.37ms
step:1208/1480 train_time:189735ms step_avg:158.38ms
step:1209/1480 train_time:189908ms step_avg:158.39ms
step:1210/1480 train_time:190083ms step_avg:158.40ms
step:1211/1480 train_time:190257ms step_avg:158.42ms
step:1212/1480 train_time:190429ms step_avg:158.43ms
step:1213/1480 train_time:190603ms step_avg:158.44ms
step:1214/1480 train_time:190780ms step_avg:158.46ms
step:1215/1480 train_time:190952ms step_avg:158.47ms
step:1216/1480 train_time:191123ms step_avg:158.48ms
step:1217/1480 train_time:191297ms step_avg:158.49ms
step:1218/1480 train_time:191467ms step_avg:158.50ms
step:1219/1480 train_time:191646ms step_avg:158.52ms
step:1220/1480 train_time:191814ms step_avg:158.52ms
step:1221/1480 train_time:191982ms step_avg:158.53ms
step:1222/1480 train_time:192149ms step_avg:158.54ms
step:1223/1480 train_time:192318ms step_avg:158.55ms
step:1224/1480 train_time:192494ms step_avg:158.56ms
step:1225/1480 train_time:192665ms step_avg:158.57ms
step:1226/1480 train_time:192839ms step_avg:158.58ms
step:1227/1480 train_time:193013ms step_avg:158.60ms
step:1228/1480 train_time:193183ms step_avg:158.61ms
step:1229/1480 train_time:193355ms step_avg:158.62ms
step:1230/1480 train_time:193533ms step_avg:158.63ms
step:1231/1480 train_time:193708ms step_avg:158.65ms
step:1232/1480 train_time:193882ms step_avg:158.66ms
step:1233/1480 train_time:194051ms step_avg:158.67ms
step:1234/1480 train_time:194223ms step_avg:158.68ms
step:1235/1480 train_time:194397ms step_avg:158.69ms
step:1236/1480 train_time:194566ms step_avg:158.70ms
step:1237/1480 train_time:194737ms step_avg:158.71ms
step:1238/1480 train_time:194924ms step_avg:158.73ms
step:1239/1480 train_time:195093ms step_avg:158.74ms
step:1240/1480 train_time:195264ms step_avg:158.75ms
step:1241/1480 train_time:195437ms step_avg:158.76ms
step:1242/1480 train_time:195607ms step_avg:158.77ms
step:1243/1480 train_time:195780ms step_avg:158.78ms
step:1244/1480 train_time:195946ms step_avg:158.79ms
step:1245/1480 train_time:196115ms step_avg:158.80ms
step:1246/1480 train_time:196285ms step_avg:158.81ms
step:1247/1480 train_time:196453ms step_avg:158.81ms
step:1248/1480 train_time:196623ms step_avg:158.82ms
step:1249/1480 train_time:196791ms step_avg:158.83ms
step:1250/1480 train_time:196960ms step_avg:158.84ms
step:1250/1480 val_loss:3.3347 train_time:197032ms step_avg:158.90ms
step:1251/1480 train_time:197138ms step_avg:158.85ms
step:1252/1480 train_time:197307ms step_avg:158.86ms
step:1253/1480 train_time:197476ms step_avg:158.87ms
step:1254/1480 train_time:197648ms step_avg:158.88ms
step:1255/1480 train_time:197833ms step_avg:158.90ms
step:1256/1480 train_time:198006ms step_avg:158.91ms
step:1257/1480 train_time:198177ms step_avg:158.92ms
step:1258/1480 train_time:198354ms step_avg:158.94ms
step:1259/1480 train_time:198525ms step_avg:158.95ms
step:1260/1480 train_time:198693ms step_avg:158.95ms
step:1261/1480 train_time:198866ms step_avg:158.97ms
step:1262/1480 train_time:199041ms step_avg:158.98ms
step:1263/1480 train_time:199215ms step_avg:158.99ms
step:1264/1480 train_time:199383ms step_avg:159.00ms
step:1265/1480 train_time:199551ms step_avg:159.00ms
step:1266/1480 train_time:199723ms step_avg:159.02ms
step:1267/1480 train_time:199893ms step_avg:159.02ms
step:1268/1480 train_time:200064ms step_avg:159.03ms
step:1269/1480 train_time:200239ms step_avg:159.05ms
step:1270/1480 train_time:200409ms step_avg:159.05ms
step:1271/1480 train_time:200577ms step_avg:159.06ms
step:1272/1480 train_time:200743ms step_avg:159.07ms
step:1273/1480 train_time:200914ms step_avg:159.08ms
step:1274/1480 train_time:201085ms step_avg:159.09ms
step:1275/1480 train_time:201254ms step_avg:159.09ms
step:1276/1480 train_time:201419ms step_avg:159.10ms
step:1277/1480 train_time:201591ms step_avg:159.11ms
step:1278/1480 train_time:201759ms step_avg:159.12ms
step:1279/1480 train_time:201931ms step_avg:159.13ms
step:1280/1480 train_time:202110ms step_avg:159.14ms
step:1281/1480 train_time:202279ms step_avg:159.15ms
step:1282/1480 train_time:202443ms step_avg:159.15ms
step:1283/1480 train_time:202613ms step_avg:159.16ms
step:1284/1480 train_time:202783ms step_avg:159.17ms
step:1285/1480 train_time:202953ms step_avg:159.18ms
step:1286/1480 train_time:203122ms step_avg:159.19ms
step:1287/1480 train_time:203293ms step_avg:159.20ms
step:1288/1480 train_time:203465ms step_avg:159.21ms
step:1289/1480 train_time:203650ms step_avg:159.23ms
step:1290/1480 train_time:203830ms step_avg:159.24ms
step:1291/1480 train_time:204003ms step_avg:159.25ms
step:1292/1480 train_time:204178ms step_avg:159.26ms
step:1293/1480 train_time:204352ms step_avg:159.28ms
step:1294/1480 train_time:204523ms step_avg:159.29ms
step:1295/1480 train_time:204694ms step_avg:159.29ms
step:1296/1480 train_time:204868ms step_avg:159.31ms
step:1297/1480 train_time:205039ms step_avg:159.32ms
step:1298/1480 train_time:205211ms step_avg:159.32ms
step:1299/1480 train_time:205380ms step_avg:159.33ms
step:1300/1480 train_time:205549ms step_avg:159.34ms
step:1301/1480 train_time:205717ms step_avg:159.35ms
step:1302/1480 train_time:205892ms step_avg:159.36ms
step:1303/1480 train_time:206070ms step_avg:159.37ms
step:1304/1480 train_time:206243ms step_avg:159.38ms
step:1305/1480 train_time:206412ms step_avg:159.39ms
step:1306/1480 train_time:206586ms step_avg:159.40ms
step:1307/1480 train_time:206754ms step_avg:159.41ms
step:1308/1480 train_time:206922ms step_avg:159.42ms
step:1309/1480 train_time:207094ms step_avg:159.43ms
step:1310/1480 train_time:207261ms step_avg:159.43ms
step:1311/1480 train_time:207430ms step_avg:159.44ms
step:1312/1480 train_time:207601ms step_avg:159.45ms
step:1313/1480 train_time:207771ms step_avg:159.46ms
step:1314/1480 train_time:207943ms step_avg:159.47ms
step:1315/1480 train_time:208115ms step_avg:159.47ms
step:1316/1480 train_time:208281ms step_avg:159.48ms
step:1317/1480 train_time:208453ms step_avg:159.49ms
step:1318/1480 train_time:208634ms step_avg:159.51ms
step:1319/1480 train_time:208810ms step_avg:159.52ms
step:1320/1480 train_time:208986ms step_avg:159.53ms
step:1321/1480 train_time:209158ms step_avg:159.54ms
step:1322/1480 train_time:209339ms step_avg:159.56ms
step:1323/1480 train_time:209511ms step_avg:159.57ms
step:1324/1480 train_time:209687ms step_avg:159.58ms
step:1325/1480 train_time:209871ms step_avg:159.60ms
step:1326/1480 train_time:210044ms step_avg:159.61ms
step:1327/1480 train_time:210214ms step_avg:159.62ms
step:1328/1480 train_time:210383ms step_avg:159.62ms
step:1329/1480 train_time:210581ms step_avg:159.65ms
step:1330/1480 train_time:210759ms step_avg:159.67ms
step:1331/1480 train_time:210929ms step_avg:159.67ms
step:1332/1480 train_time:211103ms step_avg:159.68ms
step:1333/1480 train_time:211278ms step_avg:159.70ms
step:1334/1480 train_time:211450ms step_avg:159.71ms
step:1335/1480 train_time:211618ms step_avg:159.71ms
step:1336/1480 train_time:211804ms step_avg:159.73ms
step:1337/1480 train_time:211980ms step_avg:159.74ms
step:1338/1480 train_time:212153ms step_avg:159.75ms
step:1339/1480 train_time:212328ms step_avg:159.76ms
step:1340/1480 train_time:212500ms step_avg:159.77ms
step:1341/1480 train_time:212669ms step_avg:159.78ms
step:1342/1480 train_time:212841ms step_avg:159.79ms
step:1343/1480 train_time:213011ms step_avg:159.80ms
step:1344/1480 train_time:213182ms step_avg:159.81ms
step:1345/1480 train_time:213364ms step_avg:159.82ms
step:1346/1480 train_time:213532ms step_avg:159.83ms
step:1347/1480 train_time:213703ms step_avg:159.84ms
step:1348/1480 train_time:213873ms step_avg:159.85ms
step:1349/1480 train_time:214041ms step_avg:159.85ms
step:1350/1480 train_time:214216ms step_avg:159.86ms
step:1351/1480 train_time:214387ms step_avg:159.87ms
step:1352/1480 train_time:214558ms step_avg:159.88ms
step:1353/1480 train_time:214735ms step_avg:159.89ms
step:1354/1480 train_time:214906ms step_avg:159.90ms
step:1355/1480 train_time:215074ms step_avg:159.91ms
step:1356/1480 train_time:215247ms step_avg:159.92ms
step:1357/1480 train_time:215421ms step_avg:159.93ms
step:1358/1480 train_time:215593ms step_avg:159.94ms
step:1359/1480 train_time:215766ms step_avg:159.94ms
step:1360/1480 train_time:215939ms step_avg:159.95ms
step:1361/1480 train_time:216117ms step_avg:159.97ms
step:1362/1480 train_time:216292ms step_avg:159.98ms
step:1363/1480 train_time:216474ms step_avg:160.00ms
step:1364/1480 train_time:216642ms step_avg:160.00ms
step:1365/1480 train_time:216809ms step_avg:160.01ms
step:1366/1480 train_time:216981ms step_avg:160.02ms
step:1367/1480 train_time:217153ms step_avg:160.02ms
step:1368/1480 train_time:217326ms step_avg:160.03ms
step:1369/1480 train_time:217507ms step_avg:160.05ms
step:1370/1480 train_time:217684ms step_avg:160.06ms
step:1371/1480 train_time:217856ms step_avg:160.07ms
step:1372/1480 train_time:218035ms step_avg:160.08ms
step:1373/1480 train_time:218203ms step_avg:160.09ms
step:1374/1480 train_time:218378ms step_avg:160.10ms
step:1375/1480 train_time:218551ms step_avg:160.11ms
step:1375/1480 val_loss:3.2957 train_time:218619ms step_avg:160.16ms
step:1376/1480 train_time:218726ms step_avg:160.12ms
step:1377/1480 train_time:218899ms step_avg:160.13ms
step:1378/1480 train_time:219068ms step_avg:160.14ms
step:1379/1480 train_time:219243ms step_avg:160.15ms
step:1380/1480 train_time:219418ms step_avg:160.16ms
step:1381/1480 train_time:219599ms step_avg:160.17ms
step:1382/1480 train_time:219770ms step_avg:160.18ms
step:1383/1480 train_time:219943ms step_avg:160.19ms
step:1384/1480 train_time:220121ms step_avg:160.20ms
step:1385/1480 train_time:220287ms step_avg:160.21ms
step:1386/1480 train_time:220457ms step_avg:160.22ms
step:1387/1480 train_time:220628ms step_avg:160.22ms
step:1388/1480 train_time:220796ms step_avg:160.23ms
step:1389/1480 train_time:220969ms step_avg:160.24ms
step:1390/1480 train_time:221137ms step_avg:160.24ms
step:1391/1480 train_time:221306ms step_avg:160.25ms
step:1392/1480 train_time:221480ms step_avg:160.26ms
step:1393/1480 train_time:221649ms step_avg:160.27ms
step:1394/1480 train_time:221819ms step_avg:160.27ms
step:1395/1480 train_time:221991ms step_avg:160.28ms
step:1396/1480 train_time:222157ms step_avg:160.29ms
step:1397/1480 train_time:222323ms step_avg:160.29ms
step:1398/1480 train_time:222489ms step_avg:160.29ms
step:1399/1480 train_time:222660ms step_avg:160.30ms
step:1400/1480 train_time:222837ms step_avg:160.31ms
step:1401/1480 train_time:223003ms step_avg:160.32ms
step:1402/1480 train_time:223176ms step_avg:160.33ms
step:1403/1480 train_time:223351ms step_avg:160.34ms
step:1404/1480 train_time:223522ms step_avg:160.35ms
step:1405/1480 train_time:223696ms step_avg:160.36ms
step:1406/1480 train_time:223871ms step_avg:160.37ms
step:1407/1480 train_time:224041ms step_avg:160.37ms
step:1408/1480 train_time:224210ms step_avg:160.38ms
step:1409/1480 train_time:224392ms step_avg:160.39ms
step:1410/1480 train_time:224562ms step_avg:160.40ms
step:1411/1480 train_time:224730ms step_avg:160.41ms
step:1412/1480 train_time:224900ms step_avg:160.41ms
step:1413/1480 train_time:225070ms step_avg:160.42ms
step:1414/1480 train_time:225241ms step_avg:160.43ms
step:1415/1480 train_time:225415ms step_avg:160.44ms
step:1416/1480 train_time:225602ms step_avg:160.46ms
step:1417/1480 train_time:225775ms step_avg:160.47ms
step:1418/1480 train_time:225946ms step_avg:160.47ms
step:1419/1480 train_time:226119ms step_avg:160.48ms
step:1420/1480 train_time:226293ms step_avg:160.49ms
step:1421/1480 train_time:226469ms step_avg:160.50ms
step:1422/1480 train_time:226642ms step_avg:160.51ms
step:1423/1480 train_time:226810ms step_avg:160.52ms
step:1424/1480 train_time:226987ms step_avg:160.53ms
step:1425/1480 train_time:227168ms step_avg:160.54ms
step:1426/1480 train_time:227339ms step_avg:160.55ms
step:1427/1480 train_time:227514ms step_avg:160.56ms
step:1428/1480 train_time:227685ms step_avg:160.57ms
step:1429/1480 train_time:227853ms step_avg:160.57ms
step:1430/1480 train_time:228025ms step_avg:160.58ms
step:1431/1480 train_time:228201ms step_avg:160.59ms
step:1432/1480 train_time:228378ms step_avg:160.60ms
step:1433/1480 train_time:228559ms step_avg:160.62ms
step:1434/1480 train_time:228739ms step_avg:160.63ms
step:1435/1480 train_time:228914ms step_avg:160.64ms
step:1436/1480 train_time:229088ms step_avg:160.65ms
step:1437/1480 train_time:229259ms step_avg:160.66ms
step:1438/1480 train_time:229428ms step_avg:160.66ms
step:1439/1480 train_time:229602ms step_avg:160.67ms
step:1440/1480 train_time:229773ms step_avg:160.68ms
step:1441/1480 train_time:229944ms step_avg:160.69ms
step:1442/1480 train_time:230122ms step_avg:160.70ms
step:1443/1480 train_time:230311ms step_avg:160.72ms
step:1444/1480 train_time:230483ms step_avg:160.73ms
step:1445/1480 train_time:230653ms step_avg:160.73ms
step:1446/1480 train_time:230829ms step_avg:160.74ms
step:1447/1480 train_time:231006ms step_avg:160.76ms
step:1448/1480 train_time:231179ms step_avg:160.76ms
step:1449/1480 train_time:231351ms step_avg:160.77ms
step:1450/1480 train_time:231523ms step_avg:160.78ms
step:1451/1480 train_time:231694ms step_avg:160.79ms
step:1452/1480 train_time:231868ms step_avg:160.80ms
step:1453/1480 train_time:232038ms step_avg:160.80ms
step:1454/1480 train_time:232211ms step_avg:160.81ms
step:1455/1480 train_time:232389ms step_avg:160.82ms
step:1456/1480 train_time:232562ms step_avg:160.83ms
step:1457/1480 train_time:232734ms step_avg:160.84ms
step:1458/1480 train_time:232906ms step_avg:160.85ms
step:1459/1480 train_time:233085ms step_avg:160.86ms
step:1460/1480 train_time:233257ms step_avg:160.87ms
step:1461/1480 train_time:233430ms step_avg:160.88ms
step:1462/1480 train_time:233601ms step_avg:160.88ms
step:1463/1480 train_time:233778ms step_avg:160.89ms
step:1464/1480 train_time:233952ms step_avg:160.90ms
step:1465/1480 train_time:234124ms step_avg:160.91ms
step:1466/1480 train_time:234295ms step_avg:160.92ms
step:1467/1480 train_time:234470ms step_avg:160.93ms
step:1468/1480 train_time:234641ms step_avg:160.93ms
step:1469/1480 train_time:234814ms step_avg:160.94ms
step:1470/1480 train_time:234995ms step_avg:160.96ms
step:1471/1480 train_time:235184ms step_avg:160.97ms
step:1472/1480 train_time:235364ms step_avg:160.99ms
step:1473/1480 train_time:235535ms step_avg:160.99ms
step:1474/1480 train_time:235713ms step_avg:161.01ms
step:1475/1480 train_time:235892ms step_avg:161.02ms
step:1476/1480 train_time:236064ms step_avg:161.03ms
step:1477/1480 train_time:236247ms step_avg:161.04ms
step:1478/1480 train_time:236429ms step_avg:161.05ms
step:1479/1480 train_time:236602ms step_avg:161.06ms
step:1480/1480 train_time:236773ms step_avg:161.07ms
step:1480/1480 val_loss:3.2766 train_time:236845ms step_avg:161.12ms