35c28373-6cf8-463c-8007-1146801f8068.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
import contextlib
from dataclasses import dataclass
from pathlib import Path

import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.nn.attention.flex_attention import BlockMask, flex_attention #KoszarskyB

# -----------------------------------------------------------------------------
# Muon optimizer

@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert len(G.shape) == 2
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    X /= (X.norm() + eps) # ensure top singular value <= 1
    if G.size(0) > G.size(1):
        X = X.T
    for _ in range(steps):
        A = X @ X.T
        B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    if G.size(0) > G.size(1):
        X = X.T
    return X

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven't tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        ns_steps: The number of Newton-Schulz iteration steps to use.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True, ns_steps=5):
        self.world_size = int(os.environ['WORLD_SIZE'])
        self.rank = int(os.environ['RANK'])
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, ns_steps=ns_steps)
        params = list(params)
        assert all(isinstance(p, torch.Tensor) for p in params)
        sizes = {p.numel() for p in params}
        param_groups = [
            {
                'params': [p for p in params if p.numel() == size],
                'update_buffer': [
                    torch.empty(size, device='cuda', dtype=torch.bfloat16)
                    for _ in range(self.world_size)
                ],
            }
            for size in sizes
        ]
        super().__init__(param_groups, defaults)

    def step(self):

        for group in self.param_groups:

            lr = group['lr']
            momentum = group['momentum']
            nesterov = group['nesterov']
            ns_steps = group['ns_steps']
            update_buffers = group['update_buffer']
            # generate weight updates in distributed fashion
            params = group['params']
            assert len(params) % self.world_size == 0
            handle = None
            params_world = None
            def update_prev():
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffers):
                    p_world.data.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.world_size]:
                p = params[base_i + self.rank]
                g = p.grad
                assert g is not None
                state = self.state[p]
                if 'momentum_buffer' not in state:
                    state['momentum_buffer'] = torch.zeros_like(g)
                buf = state['momentum_buffer']
                buf.lerp_(g, 1 - momentum)
                g = g.lerp_(buf, momentum) if nesterov else buf
                g = zeropower_via_newtonschulz5(g, steps=ns_steps).flatten()
                update_prev()
                handle = dist.all_gather(update_buffers, g, async_op=True)
                params_world = params[base_i : base_i + self.world_size]
            update_prev()

# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):

    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=False)

    def forward(self, x):
        return F.linear(x, self.weight.to(x.dtype))

class Rotary(torch.nn.Module):

    def __init__(self, dim, base=10000):
        super().__init__()
        self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None

    def forward(self, x):
        seq_len = x.shape[1]
        if seq_len != self.seq_len_cached:
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.outer(t, self.inv_freq)
            self.seq_len_cached = seq_len
            self.cos_cached = freqs.cos()
            self.sin_cached = freqs.sin()
        cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
        # apply_rotary_emb(x, cos, sin)
        x1, x2 = x.chunk(2, dim=3)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x)

class CausalSelfAttention(nn.Module):

    def __init__(self, dim, num_heads):
        super().__init__()
        assert dim % num_heads == 0
        self.num_heads = num_heads
        self.c_q = CastedLinear(dim, dim)
        self.c_k = CastedLinear(dim, dim)
        self.c_v = CastedLinear(dim, dim)
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5]))
        self.rotary = Rotary(dim // num_heads) # dim // num_heads = head_dim
        self.c_proj = CastedLinear(dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x, vi, block_mask):
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q = self.c_q(x).view(B, T, self.num_heads, -1)
        k = self.c_k(x).view(B, T, self.num_heads, -1)
        v = self.c_v(x).view(B, T, self.num_heads, -1)
        v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @KoszarskyB & @Grad62304977
        q, k = norm(q), norm(k) # QK norm @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask, enable_gqa=True)
        y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):

    def __init__(self, dim):
        super().__init__()
        self.c_fc   = CastedLinear(dim, 4 * dim)
        self.c_proj = CastedLinear(4 * dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x):
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):

    def __init__(self, config):
        super().__init__()
        self.attn = CausalSelfAttention(config.model_dim, config.num_heads)
        self.mlp = MLP(config.model_dim)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x, vi, x0, block_mask):
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        x = x + self.attn(norm(x), vi, block_mask)
        x = x + self.mlp(norm(x))
        return x

class ValueEmbedding(nn.Module):
    def __init__(self, config: "GPTConfig"):
        super().__init__()
        self.__setattr__
        self.embed = nn.ModuleList([
            nn.Embedding(config.vocab_size, config.model_dim)
            for _ in range(6)
        ])

    def forward(self, inputs) -> "list[torch.Tensor]":
        ve = [emb(inputs) for emb in self.embed]
        ve += reversed(ve)
        return ve


# -----------------------------------------------------------------------------
# The main GPT-2 model

@dataclass
class GPTConfig:
    vocab_size : int = 50304
    num_layers : int = 12
    num_heads : int = 6 # head dim 128 suggested by @Grad62304977
    model_dim : int = 768

class GPT(nn.Module):

    def __init__(self, config: GPTConfig):
        super().__init__()
        self.num_layers = config.num_layers

        # U-net design by @brendanh0gan
        self.num_encoder_layers = config.num_layers // 2 # Half of the layers for encoder
        self.num_decoder_layers = config.num_layers - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))

        self.embed = nn.Embedding(config.vocab_size, config.model_dim)
        self.blocks = nn.ModuleList([Block(config) for _ in range(config.num_layers)])
        # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
        # U-net structure on token value embeddings by @leloykun
        self.value_embeds = ValueEmbedding(config)
        self.lm_head = CastedLinear(config.model_dim, config.vocab_size)
        self.lm_head.weight.data.zero_() # @Grad62304977

    def forward(
        self,
        inputs: torch.Tensor,
        targets: torch.Tensor,
        sliding_window_num_blocks: torch.Tensor,
    ):
        BLOCK_SIZE = 128
        assert inputs.ndim == 1
        docs = (inputs == 50256).cumsum(0)
        docs_low = docs.view(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.view(-1, BLOCK_SIZE)[:, -1].contiguous()

        def document_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            return causal_mask & document_mask

        def dense_to_ordered(dense_mask: torch.Tensor):
            num_blocks = dense_mask.sum(dim=-1, dtype=torch.int32)
            indices = dense_mask.argsort(dim=-1, descending=True, stable=True).to(torch.int32)
            return num_blocks[None, None].contiguous(), indices[None, None].contiguous()

        def create_doc_swc_block_mask(sliding_window_num_blocks: torch.Tensor):
            kv_idx = block_idx = torch.arange(512, dtype=torch.int32, device="cuda")
            q_idx = block_idx[:, None]
            causal_bm = q_idx >= kv_idx
            causal_full_bm = q_idx > kv_idx
            window_bm = q_idx - kv_idx < sliding_window_num_blocks
            window_full_bm = window_bm
            # document_bm = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
            document_bm = (docs_low[:, None] <= docs_high) & (docs_low <= docs_high[:, None])
            document_full_bm = (docs_low[:, None] == docs_high) & (docs_low == docs_high[:, None])
            nonzero_bm = causal_bm & window_bm & document_bm
            full_bm  = causal_full_bm & window_full_bm & document_full_bm
            kv_num_blocks, kv_indices = dense_to_ordered(nonzero_bm ^ full_bm)
            full_kv_num_blocks, full_kv_indices = dense_to_ordered(full_bm)
            return BlockMask.from_kv_blocks(
                kv_num_blocks,
                kv_indices,
                full_kv_num_blocks,
                full_kv_indices,
                BLOCK_SIZE=BLOCK_SIZE,
                mask_mod=document_causal,
            )

        block_mask = create_doc_swc_block_mask(sliding_window_num_blocks)

        # forward the GPT model itself
        x = self.embed(inputs[None]) # token embeddings of shape (b, t, model_dim)
        x = norm(x) # @Grad62304977
        x0 = x
        ve = self.value_embeds(inputs)
        ve_enc, ve_dec = ve[:self.num_encoder_layers], ve[self.num_encoder_layers:]

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        for i in range(self.num_encoder_layers):
            x = self.blocks[i](x, ve_enc[i], x0, block_mask)
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            # U-net structure on token value embeddings by @leloykun
            x = self.blocks[self.num_encoder_layers + i](x, ve_dec[i], x0, block_mask)

        x = norm(x)
        logits = self.lm_head(x)
        logits = 30 * torch.tanh(logits / 30) # @Grad62304977
        logits = logits.float()
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1))
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _peek_data_shard(file: Path):
    # only reads the header, returns header data
    # header is 256 int32
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    return int(header[2]) # number of tokens (claimed)

def _load_data_shard(path: Path, num_tokens):
    with path.open("rb", buffering=0) as f:
        tokens = torch.empty(num_tokens, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy())
        assert nbytes == 2 * num_tokens, "number of tokens read does not match header?"
    return tokens

class DistributedDataLoader:
    def __init__(self, filename_pattern, seq_len, process_rank, num_processes):
        self.process_rank = process_rank
        self.num_processes = num_processes
        self.seq_len = seq_len

        # glob files that match the pattern
        self.files = sorted(Path.cwd().glob(filename_pattern))
        assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"

        # load and validate all data shards, count number of tokens in total
        self.files_num_tokens = [_peek_data_shard(file) for file in self.files]
        assert min(self.files_num_tokens) >= num_processes * seq_len + 1
        self.total_num_tokens = sum(self.files_num_tokens)

        self.reset()

    def reset(self):
        self.current_shard = -1
        self.advance()

    def advance(self): # advance to next data shard
        self.current_shard = (self.current_shard + 1) % len(self.files)
        self.current_position = self.process_rank * self.seq_len
        self.tokens = _load_data_shard(self.files[self.current_shard], self.files_num_tokens[self.current_shard])

    def next_batch(self):
        batch_size = self.seq_len * self.num_processes
        buf = self.tokens[self.current_position:self.current_position+self.seq_len+1]
        # host side async is sufficient;
        # no performance improvement was observed when introducing a separate stream.
        inputs = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
        targets = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
        # advance current position and load next shard if necessary
        self.current_position += batch_size
        if self.current_position + batch_size + 1 >= len(self.tokens):
            self.advance()
        return inputs, targets

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data hyperparams
    input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
    input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
    # optimization hyperparams
    batch_size : int = 8 # batch size, in sequences, across all devices
    sequence_length : int = 64*1024 # sequence length, in tokens
    num_iterations : int = 1480 # number of iterations to run
    warmup_iters : int = 0
    cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
    weight_decay : float = 0
    # evaluation and logging hyperparams
    val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
    val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()

# set up DDP (distributed data parallel). torchrun sets this env variable
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
assert torch.cuda.is_available()
device = torch.device(f"cuda:{ddp_local_rank}")
torch.cuda.set_device(device)
print(f"using device: {device}")
dist.init_process_group(backend='nccl', device_id=device)
dist.barrier()
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = uuid.uuid4()
    logdir = Path("logs") / f"{run_id}"
    logdir.mkdir(exist_ok=True)
    logfile = Path("logs") / f"{run_id}.txt"
    print(logfile.stem)
    # create the log file
    with logfile.open("w") as f:
        # begin the log by printing this file (the Python code)
        print(code, file=f)
        print("=" * 100, file=f)
def print0(s, logonly=False):
    if master_process:
        with logfile.open("a") as f:
            if not logonly:
                print(s)
            print(s, file=f)
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running python {sys.version}")
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)

# calculate the number of steps to take in the val loop.
assert args.val_tokens % (args.sequence_length * ddp_world_size) == 0
val_steps = args.val_tokens // (args.sequence_length * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size

# load tokens
train_loader = DistributedDataLoader(args.input_bin, args.sequence_length, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, args.sequence_length, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.total_num_tokens} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.total_num_tokens} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
inputs_train, targets_train = train_loader.next_batch()

# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, num_layers=12, num_heads=6, model_dim=768))
model = model.cuda().bfloat16()
for m in model.modules():
    if isinstance(m, CastedLinear):
        m.float()
config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank], broadcast_buffers=False, gradient_as_bucket_view=True)
raw_model = model.module # always contains the "raw" unwrapped model

# init the optimizer(s)
embed_params = [*raw_model.embed.parameters(), *raw_model.value_embeds.parameters()]
optimizer1 = torch.optim.Adam(embed_params, lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.blocks.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
    assert it <= args.num_iterations
    # 1) linear warmup for warmup_iters steps
    if it < args.warmup_iters:
        return (it+1) / args.warmup_iters
    # 2) constant lr for a while
    elif it < args.num_iterations - args.cooldown_iters:
        return 1.0
    # 3) linear cooldown
    else:
        decay_ratio = (args.num_iterations - it) / args.cooldown_iters
        return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]

sliding_window_num_blocks = torch.tensor(1, dtype=torch.int32, device="cuda")
sw_num_blocks_prev = 1
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
    last_step = (step == args.num_iterations)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Linearly increase the sliding window size over training in chunks of 64 from 64 -> 1792. By @fernbear.bsky.social
    frac_done = step / args.num_iterations # training progress
    sw_num_blocks = int(((1 - frac_done) * 64 + frac_done * 1792 + 64) // 128)
    if sw_num_blocks != sw_num_blocks_prev:
        sliding_window_num_blocks.copy_(sw_num_blocks, non_blocking=True)
        sw_num_blocks_prev = sw_num_blocks

    # once in a while evaluate the validation dataset
    if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # run validation batches
        model.eval()
        val_loader.reset()
        val_loss = 0.0
        for _ in range(val_steps):
            with torch.no_grad():
                inputs_val, targets_val = val_loader.next_batch()
                val_loss += model(inputs_val, targets_val, sliding_window_num_blocks)
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        val_loss /= val_steps
        # log val loss to console and to logfile
        print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # save the state of the training process
        log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
        torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    # bit confusing: we want to make sure to eval on 0th iteration
    # but also after the very last iteration. so we loop for step <= num_iterations
    # instead of just < num_iterations (one extra due to <=), only to do
    # the validation/sampling one last time, and then we break right here as we're done.
    if last_step:
        break

    # --------------- TRAINING SECTION BEGIN -----------------
    model.train()
    for i in range(1, train_accumulation_steps + 1):
        with contextlib.ExitStack() as stack:
            if i < train_accumulation_steps: # there's no need to sync gradients every accumulation step
                stack.enter_context(model.no_sync())
            if step >= 5:
                stack.enter_context(torch.compiler.set_stance(skip_guard_eval_unsafe=True))
            model(inputs_train, targets_train, sliding_window_num_blocks).backward()
            inputs_train, targets_train = train_loader.next_batch()
    if train_accumulation_steps != 1:
        for p in model.parameters():
            p.grad /= train_accumulation_steps
    # momentum warmup for Muon
    frac = min(step/300, 1)
    for group in optimizer3.param_groups:
        group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # --------------- TRAINING SECTION END -------------------
    # everything that follows now is just diagnostics, prints, logging, etc.
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")

print0(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")

# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()

====================================================================================================
Running python 3.10.12 (main, Sep 11 2024, 15:47:36) [GCC 11.4.0]
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Wed Dec 11 07:54:14 2024       
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.183.06             Driver Version: 535.183.06   CUDA Version: 12.2     |
|-----------------------------------------+----------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
|                                         |                      |               MIG M. |
|=========================================+======================+======================|
|   0  NVIDIA H100 80GB HBM3          On  | 00000000:19:00.0 Off |                    0 |
| N/A   37C    P0             125W / 700W |   7084MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  | 00000000:3B:00.0 Off |                    0 |
| N/A   30C    P0             116W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  | 00000000:4C:00.0 Off |                    0 |
| N/A   28C    P0             112W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  | 00000000:5D:00.0 Off |                    0 |
| N/A   36C    P0             114W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  | 00000000:9B:00.0 Off |                    0 |
| N/A   38C    P0             119W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  | 00000000:BB:00.0 Off |                    0 |
| N/A   30C    P0             117W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  | 00000000:CB:00.0 Off |                    0 |
| N/A   35C    P0             119W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  | 00000000:DB:00.0 Off |                    0 |
| N/A   30C    P0             118W / 700W |   3211MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
                                                                                         
+---------------------------------------------------------------------------------------+
| Processes:                                                                            |
|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
|        ID   ID                                                             Usage      |
|=======================================================================================|
+---------------------------------------------------------------------------------------+

====================================================================================================
Training DataLoader: total number of tokens: 1000000000 across 10 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:29110ms step_avg:nanms
step:2/1480 train_time:29215ms step_avg:nanms
step:3/1480 train_time:29335ms step_avg:nanms
step:4/1480 train_time:29477ms step_avg:nanms
step:5/1480 train_time:29618ms step_avg:nanms
step:6/1480 train_time:29761ms step_avg:nanms
step:7/1480 train_time:29903ms step_avg:nanms
step:8/1480 train_time:30045ms step_avg:nanms
step:9/1480 train_time:30190ms step_avg:nanms
step:10/1480 train_time:30332ms step_avg:nanms
step:11/1480 train_time:143ms step_avg:nanms
step:12/1480 train_time:284ms step_avg:nanms
step:13/1480 train_time:427ms step_avg:142.25ms
step:14/1480 train_time:569ms step_avg:142.19ms
step:15/1480 train_time:710ms step_avg:141.98ms
step:16/1480 train_time:852ms step_avg:142.00ms
step:17/1480 train_time:996ms step_avg:142.24ms
step:18/1480 train_time:1141ms step_avg:142.59ms
step:19/1480 train_time:1284ms step_avg:142.70ms
step:20/1480 train_time:1426ms step_avg:142.58ms
step:21/1480 train_time:1568ms step_avg:142.52ms
step:22/1480 train_time:1709ms step_avg:142.40ms
step:23/1480 train_time:1850ms step_avg:142.33ms
step:24/1480 train_time:1993ms step_avg:142.35ms
step:25/1480 train_time:2136ms step_avg:142.41ms
step:26/1480 train_time:2279ms step_avg:142.42ms
step:27/1480 train_time:2422ms step_avg:142.48ms
step:28/1480 train_time:2565ms step_avg:142.50ms
step:29/1480 train_time:2706ms step_avg:142.43ms
step:30/1480 train_time:2850ms step_avg:142.49ms
step:31/1480 train_time:2992ms step_avg:142.45ms
step:32/1480 train_time:3135ms step_avg:142.50ms
step:33/1480 train_time:3278ms step_avg:142.52ms
step:34/1480 train_time:3422ms step_avg:142.59ms
step:35/1480 train_time:3565ms step_avg:142.58ms
step:36/1480 train_time:3705ms step_avg:142.52ms
step:37/1480 train_time:3847ms step_avg:142.49ms
step:38/1480 train_time:3989ms step_avg:142.48ms
step:39/1480 train_time:4132ms step_avg:142.49ms
step:40/1480 train_time:4640ms step_avg:154.66ms
step:41/1480 train_time:4742ms step_avg:152.97ms
step:42/1480 train_time:4884ms step_avg:152.62ms
step:43/1480 train_time:5026ms step_avg:152.31ms
step:44/1480 train_time:5168ms step_avg:151.99ms
step:45/1480 train_time:5309ms step_avg:151.68ms
step:46/1480 train_time:5451ms step_avg:151.41ms
step:47/1480 train_time:5596ms step_avg:151.24ms
step:48/1480 train_time:5740ms step_avg:151.05ms
step:49/1480 train_time:5884ms step_avg:150.87ms
step:50/1480 train_time:6027ms step_avg:150.68ms
step:51/1480 train_time:6169ms step_avg:150.46ms
step:52/1480 train_time:6310ms step_avg:150.24ms
step:53/1480 train_time:6453ms step_avg:150.07ms
step:54/1480 train_time:6596ms step_avg:149.91ms
step:55/1480 train_time:6740ms step_avg:149.79ms
step:56/1480 train_time:6884ms step_avg:149.64ms
step:57/1480 train_time:7026ms step_avg:149.49ms
step:58/1480 train_time:7168ms step_avg:149.34ms
step:59/1480 train_time:7310ms step_avg:149.18ms
step:60/1480 train_time:7452ms step_avg:149.04ms
step:61/1480 train_time:7595ms step_avg:148.92ms
step:62/1480 train_time:7740ms step_avg:148.85ms
step:63/1480 train_time:7885ms step_avg:148.78ms
step:64/1480 train_time:8028ms step_avg:148.67ms
step:65/1480 train_time:8170ms step_avg:148.54ms
step:66/1480 train_time:8312ms step_avg:148.42ms
step:67/1480 train_time:8455ms step_avg:148.33ms
step:68/1480 train_time:8599ms step_avg:148.26ms
step:69/1480 train_time:8744ms step_avg:148.20ms
step:70/1480 train_time:8887ms step_avg:148.12ms
step:71/1480 train_time:9028ms step_avg:148.00ms
step:72/1480 train_time:9171ms step_avg:147.91ms
step:73/1480 train_time:9313ms step_avg:147.82ms
step:74/1480 train_time:9455ms step_avg:147.74ms
step:75/1480 train_time:9597ms step_avg:147.65ms
step:76/1480 train_time:9741ms step_avg:147.59ms
step:77/1480 train_time:9886ms step_avg:147.55ms
step:78/1480 train_time:10028ms step_avg:147.48ms
step:79/1480 train_time:10171ms step_avg:147.40ms
step:80/1480 train_time:10692ms step_avg:152.74ms
step:81/1480 train_time:10794ms step_avg:152.02ms
step:82/1480 train_time:10937ms step_avg:151.90ms
step:83/1480 train_time:11080ms step_avg:151.77ms
step:84/1480 train_time:11224ms step_avg:151.67ms
step:85/1480 train_time:11366ms step_avg:151.55ms
step:86/1480 train_time:11507ms step_avg:151.41ms
step:87/1480 train_time:11652ms step_avg:151.32ms
step:88/1480 train_time:11796ms step_avg:151.23ms
step:89/1480 train_time:11941ms step_avg:151.15ms
step:90/1480 train_time:12085ms step_avg:151.06ms
step:91/1480 train_time:12228ms step_avg:150.96ms
step:92/1480 train_time:12370ms step_avg:150.86ms
step:93/1480 train_time:12512ms step_avg:150.75ms
step:94/1480 train_time:12655ms step_avg:150.66ms
step:95/1480 train_time:12799ms step_avg:150.57ms
step:96/1480 train_time:12942ms step_avg:150.49ms
step:97/1480 train_time:13464ms step_avg:154.76ms
step:98/1480 train_time:13568ms step_avg:154.18ms
step:99/1480 train_time:13710ms step_avg:154.04ms
step:100/1480 train_time:13851ms step_avg:153.90ms
step:101/1480 train_time:13996ms step_avg:153.81ms
step:102/1480 train_time:14137ms step_avg:153.66ms
step:103/1480 train_time:14279ms step_avg:153.54ms
step:104/1480 train_time:14423ms step_avg:153.44ms
step:105/1480 train_time:14568ms step_avg:153.35ms
step:106/1480 train_time:14709ms step_avg:153.22ms
step:107/1480 train_time:14851ms step_avg:153.10ms
step:108/1480 train_time:14993ms step_avg:152.99ms
step:109/1480 train_time:15135ms step_avg:152.88ms
step:110/1480 train_time:15277ms step_avg:152.77ms
step:111/1480 train_time:15423ms step_avg:152.70ms
step:112/1480 train_time:15568ms step_avg:152.63ms
step:113/1480 train_time:15713ms step_avg:152.55ms
step:114/1480 train_time:15860ms step_avg:152.50ms
step:115/1480 train_time:16006ms step_avg:152.44ms
step:116/1480 train_time:16152ms step_avg:152.37ms
step:117/1480 train_time:16298ms step_avg:152.32ms
step:118/1480 train_time:16444ms step_avg:152.26ms
step:119/1480 train_time:16589ms step_avg:152.19ms
step:120/1480 train_time:16734ms step_avg:152.12ms
step:121/1480 train_time:16879ms step_avg:152.06ms
step:122/1480 train_time:17026ms step_avg:152.01ms
step:123/1480 train_time:17170ms step_avg:151.95ms
step:124/1480 train_time:17315ms step_avg:151.88ms
step:125/1480 train_time:17462ms step_avg:151.85ms
step:125/1480 val_loss:4.3910 train_time:17527ms step_avg:152.41ms
step:126/1480 train_time:17622ms step_avg:151.91ms
step:127/1480 train_time:17765ms step_avg:151.84ms
step:128/1480 train_time:17912ms step_avg:151.80ms
step:129/1480 train_time:18058ms step_avg:151.74ms
step:130/1480 train_time:18202ms step_avg:151.69ms
step:131/1480 train_time:18346ms step_avg:151.62ms
step:132/1480 train_time:18491ms step_avg:151.57ms
step:133/1480 train_time:18640ms step_avg:151.54ms
step:134/1480 train_time:18785ms step_avg:151.49ms
step:135/1480 train_time:18931ms step_avg:151.45ms
step:136/1480 train_time:19078ms step_avg:151.41ms
step:137/1480 train_time:19224ms step_avg:151.37ms
step:138/1480 train_time:19369ms step_avg:151.32ms
step:139/1480 train_time:19515ms step_avg:151.28ms
step:140/1480 train_time:19660ms step_avg:151.23ms
step:141/1480 train_time:19806ms step_avg:151.19ms
step:142/1480 train_time:19953ms step_avg:151.16ms
step:143/1480 train_time:20099ms step_avg:151.12ms
step:144/1480 train_time:20244ms step_avg:151.08ms
step:145/1480 train_time:20390ms step_avg:151.04ms
step:146/1480 train_time:20537ms step_avg:151.01ms
step:147/1480 train_time:20682ms step_avg:150.96ms
step:148/1480 train_time:20826ms step_avg:150.91ms
step:149/1480 train_time:20972ms step_avg:150.88ms
step:150/1480 train_time:21119ms step_avg:150.85ms
step:151/1480 train_time:21263ms step_avg:150.80ms
step:152/1480 train_time:21408ms step_avg:150.76ms
step:153/1480 train_time:21555ms step_avg:150.73ms
step:154/1480 train_time:21701ms step_avg:150.70ms
step:155/1480 train_time:21845ms step_avg:150.66ms
step:156/1480 train_time:21991ms step_avg:150.62ms
step:157/1480 train_time:22137ms step_avg:150.59ms
step:158/1480 train_time:22283ms step_avg:150.56ms
step:159/1480 train_time:22428ms step_avg:150.52ms
step:160/1480 train_time:22574ms step_avg:150.49ms
step:161/1480 train_time:22721ms step_avg:150.47ms
step:162/1480 train_time:22866ms step_avg:150.43ms
step:163/1480 train_time:23012ms step_avg:150.40ms
step:164/1480 train_time:23157ms step_avg:150.37ms
step:165/1480 train_time:23303ms step_avg:150.34ms
step:166/1480 train_time:23448ms step_avg:150.31ms
step:167/1480 train_time:23594ms step_avg:150.28ms
step:168/1480 train_time:23742ms step_avg:150.26ms
step:169/1480 train_time:23887ms step_avg:150.23ms
step:170/1480 train_time:24033ms step_avg:150.21ms
step:171/1480 train_time:24180ms step_avg:150.18ms
step:172/1480 train_time:24324ms step_avg:150.15ms
step:173/1480 train_time:24469ms step_avg:150.12ms
step:174/1480 train_time:24616ms step_avg:150.10ms
step:175/1480 train_time:24761ms step_avg:150.07ms
step:176/1480 train_time:24907ms step_avg:150.04ms
step:177/1480 train_time:25054ms step_avg:150.02ms
step:178/1480 train_time:25200ms step_avg:150.00ms
step:179/1480 train_time:25345ms step_avg:149.97ms
step:180/1480 train_time:25897ms step_avg:152.34ms
step:181/1480 train_time:26001ms step_avg:152.05ms
step:182/1480 train_time:26147ms step_avg:152.02ms
step:183/1480 train_time:26293ms step_avg:151.98ms
step:184/1480 train_time:26440ms step_avg:151.95ms
step:185/1480 train_time:26585ms step_avg:151.91ms
step:186/1480 train_time:26730ms step_avg:151.88ms
step:187/1480 train_time:26877ms step_avg:151.85ms
step:188/1480 train_time:27024ms step_avg:151.82ms
step:189/1480 train_time:27184ms step_avg:151.87ms
step:190/1480 train_time:27318ms step_avg:151.77ms
step:191/1480 train_time:27463ms step_avg:151.73ms
step:192/1480 train_time:27608ms step_avg:151.69ms
step:193/1480 train_time:27754ms step_avg:151.66ms
step:194/1480 train_time:27901ms step_avg:151.64ms
step:195/1480 train_time:28046ms step_avg:151.60ms
step:196/1480 train_time:28193ms step_avg:151.58ms
step:197/1480 train_time:28340ms step_avg:151.55ms
step:198/1480 train_time:28486ms step_avg:151.52ms
step:199/1480 train_time:28631ms step_avg:151.49ms
step:200/1480 train_time:28778ms step_avg:151.46ms
step:201/1480 train_time:28924ms step_avg:151.43ms
step:202/1480 train_time:29069ms step_avg:151.40ms
step:203/1480 train_time:29215ms step_avg:151.37ms
step:204/1480 train_time:29361ms step_avg:151.34ms
step:205/1480 train_time:29507ms step_avg:151.32ms
step:206/1480 train_time:29652ms step_avg:151.29ms
step:207/1480 train_time:29799ms step_avg:151.26ms
step:208/1480 train_time:29944ms step_avg:151.23ms
step:209/1480 train_time:30091ms step_avg:151.21ms
step:210/1480 train_time:30237ms step_avg:151.19ms
step:211/1480 train_time:30382ms step_avg:151.15ms
step:212/1480 train_time:30526ms step_avg:151.12ms
step:213/1480 train_time:30672ms step_avg:151.10ms
step:214/1480 train_time:30819ms step_avg:151.07ms
step:215/1480 train_time:30963ms step_avg:151.04ms
step:216/1480 train_time:31109ms step_avg:151.01ms
step:217/1480 train_time:31255ms step_avg:150.99ms
step:218/1480 train_time:31401ms step_avg:150.96ms
step:219/1480 train_time:31546ms step_avg:150.94ms
step:220/1480 train_time:31692ms step_avg:150.91ms
step:221/1480 train_time:32220ms step_avg:152.70ms
step:222/1480 train_time:32733ms step_avg:154.40ms
step:223/1480 train_time:32843ms step_avg:154.19ms
step:224/1480 train_time:32991ms step_avg:154.16ms
step:225/1480 train_time:33139ms step_avg:154.14ms
step:226/1480 train_time:33286ms step_avg:154.10ms
step:227/1480 train_time:33435ms step_avg:154.08ms
step:228/1480 train_time:33582ms step_avg:154.05ms
step:229/1480 train_time:33730ms step_avg:154.02ms
step:230/1480 train_time:33879ms step_avg:154.00ms
step:231/1480 train_time:34026ms step_avg:153.97ms
step:232/1480 train_time:34175ms step_avg:153.94ms
step:233/1480 train_time:34323ms step_avg:153.92ms
step:234/1480 train_time:34472ms step_avg:153.89ms
step:235/1480 train_time:34620ms step_avg:153.87ms
step:236/1480 train_time:34768ms step_avg:153.84ms
step:237/1480 train_time:34917ms step_avg:153.82ms
step:238/1480 train_time:35064ms step_avg:153.79ms
step:239/1480 train_time:35214ms step_avg:153.77ms
step:240/1480 train_time:35362ms step_avg:153.75ms
step:241/1480 train_time:35510ms step_avg:153.72ms
step:242/1480 train_time:35658ms step_avg:153.70ms
step:243/1480 train_time:35806ms step_avg:153.67ms
step:244/1480 train_time:35956ms step_avg:153.66ms
step:245/1480 train_time:36104ms step_avg:153.63ms
step:246/1480 train_time:36253ms step_avg:153.61ms
step:247/1480 train_time:36402ms step_avg:153.59ms
step:248/1480 train_time:36549ms step_avg:153.57ms
step:249/1480 train_time:36698ms step_avg:153.55ms
step:250/1480 train_time:36845ms step_avg:153.52ms
step:250/1480 val_loss:3.9817 train_time:36912ms step_avg:153.80ms
step:251/1480 train_time:37007ms step_avg:153.56ms
step:252/1480 train_time:37151ms step_avg:153.52ms
step:253/1480 train_time:37301ms step_avg:153.50ms
step:254/1480 train_time:37448ms step_avg:153.48ms
step:255/1480 train_time:37596ms step_avg:153.45ms
step:256/1480 train_time:37744ms step_avg:153.43ms
step:257/1480 train_time:37892ms step_avg:153.41ms
step:258/1480 train_time:38041ms step_avg:153.39ms
step:259/1480 train_time:38190ms step_avg:153.38ms
step:260/1480 train_time:38340ms step_avg:153.36ms
step:261/1480 train_time:38487ms step_avg:153.34ms
step:262/1480 train_time:38636ms step_avg:153.32ms
step:263/1480 train_time:38783ms step_avg:153.29ms
step:264/1480 train_time:38932ms step_avg:153.27ms
step:265/1480 train_time:39080ms step_avg:153.26ms
step:266/1480 train_time:39228ms step_avg:153.24ms
step:267/1480 train_time:39378ms step_avg:153.22ms
step:268/1480 train_time:39525ms step_avg:153.20ms
step:269/1480 train_time:39675ms step_avg:153.18ms
step:270/1480 train_time:39822ms step_avg:153.16ms
step:271/1480 train_time:39970ms step_avg:153.14ms
step:272/1480 train_time:40119ms step_avg:153.13ms
step:273/1480 train_time:40266ms step_avg:153.10ms
step:274/1480 train_time:40415ms step_avg:153.09ms
step:275/1480 train_time:40563ms step_avg:153.07ms
step:276/1480 train_time:40711ms step_avg:153.05ms
step:277/1480 train_time:40860ms step_avg:153.03ms
step:278/1480 train_time:41008ms step_avg:153.01ms
step:279/1480 train_time:41157ms step_avg:153.00ms
step:280/1480 train_time:41304ms step_avg:152.98ms
step:281/1480 train_time:41454ms step_avg:152.97ms
step:282/1480 train_time:41602ms step_avg:152.95ms
step:283/1480 train_time:41750ms step_avg:152.93ms
step:284/1480 train_time:41899ms step_avg:152.92ms
step:285/1480 train_time:42048ms step_avg:152.90ms
step:286/1480 train_time:42197ms step_avg:152.89ms
step:287/1480 train_time:42344ms step_avg:152.87ms
step:288/1480 train_time:42493ms step_avg:152.85ms
step:289/1480 train_time:42641ms step_avg:152.83ms
step:290/1480 train_time:42789ms step_avg:152.82ms
step:291/1480 train_time:42938ms step_avg:152.80ms
step:292/1480 train_time:43086ms step_avg:152.79ms
step:293/1480 train_time:43235ms step_avg:152.77ms
step:294/1480 train_time:43382ms step_avg:152.75ms
step:295/1480 train_time:43531ms step_avg:152.74ms
step:296/1480 train_time:43681ms step_avg:152.73ms
step:297/1480 train_time:43828ms step_avg:152.71ms
step:298/1480 train_time:43978ms step_avg:152.70ms
step:299/1480 train_time:44126ms step_avg:152.68ms
step:300/1480 train_time:44276ms step_avg:152.68ms
step:301/1480 train_time:44423ms step_avg:152.66ms
step:302/1480 train_time:44571ms step_avg:152.64ms
step:303/1480 train_time:44721ms step_avg:152.63ms
step:304/1480 train_time:44868ms step_avg:152.61ms
step:305/1480 train_time:45017ms step_avg:152.60ms
step:306/1480 train_time:45164ms step_avg:152.58ms
step:307/1480 train_time:45313ms step_avg:152.57ms
step:308/1480 train_time:45461ms step_avg:152.55ms
step:309/1480 train_time:45609ms step_avg:152.54ms
step:310/1480 train_time:45758ms step_avg:152.53ms
step:311/1480 train_time:45905ms step_avg:152.51ms
step:312/1480 train_time:46055ms step_avg:152.50ms
step:313/1480 train_time:46203ms step_avg:152.48ms
step:314/1480 train_time:46352ms step_avg:152.47ms
step:315/1480 train_time:46501ms step_avg:152.46ms
step:316/1480 train_time:46649ms step_avg:152.45ms
step:317/1480 train_time:46799ms step_avg:152.44ms
step:318/1480 train_time:46948ms step_avg:152.43ms
step:319/1480 train_time:47097ms step_avg:152.42ms
step:320/1480 train_time:47244ms step_avg:152.40ms
step:321/1480 train_time:47394ms step_avg:152.39ms
step:322/1480 train_time:47542ms step_avg:152.38ms
step:323/1480 train_time:47690ms step_avg:152.36ms
step:324/1480 train_time:47839ms step_avg:152.35ms
step:325/1480 train_time:47987ms step_avg:152.34ms
step:326/1480 train_time:48136ms step_avg:152.33ms
step:327/1480 train_time:48283ms step_avg:152.31ms
step:328/1480 train_time:48433ms step_avg:152.30ms
step:329/1480 train_time:48581ms step_avg:152.29ms
step:330/1480 train_time:48730ms step_avg:152.28ms
step:331/1480 train_time:48881ms step_avg:152.28ms
step:332/1480 train_time:49032ms step_avg:152.27ms
step:333/1480 train_time:49183ms step_avg:152.27ms
step:334/1480 train_time:49334ms step_avg:152.27ms
step:335/1480 train_time:49484ms step_avg:152.26ms
step:336/1480 train_time:49636ms step_avg:152.26ms
step:337/1480 train_time:49786ms step_avg:152.25ms
step:338/1480 train_time:49938ms step_avg:152.25ms
step:339/1480 train_time:50087ms step_avg:152.24ms
step:340/1480 train_time:50239ms step_avg:152.24ms
step:341/1480 train_time:50389ms step_avg:152.23ms
step:342/1480 train_time:50540ms step_avg:152.23ms
step:343/1480 train_time:50690ms step_avg:152.22ms
step:344/1480 train_time:50841ms step_avg:152.22ms
step:345/1480 train_time:50991ms step_avg:152.21ms
step:346/1480 train_time:51142ms step_avg:152.21ms
step:347/1480 train_time:51293ms step_avg:152.21ms
step:348/1480 train_time:51444ms step_avg:152.20ms
step:349/1480 train_time:51596ms step_avg:152.20ms
step:350/1480 train_time:51746ms step_avg:152.19ms
step:351/1480 train_time:51897ms step_avg:152.19ms
step:352/1480 train_time:52048ms step_avg:152.19ms
step:353/1480 train_time:52199ms step_avg:152.18ms
step:354/1480 train_time:52350ms step_avg:152.18ms
step:355/1480 train_time:52500ms step_avg:152.17ms
step:356/1480 train_time:52651ms step_avg:152.17ms
step:357/1480 train_time:52801ms step_avg:152.17ms
step:358/1480 train_time:52952ms step_avg:152.16ms
step:359/1480 train_time:53103ms step_avg:152.16ms
step:360/1480 train_time:53255ms step_avg:152.16ms
step:361/1480 train_time:53406ms step_avg:152.15ms
step:362/1480 train_time:53557ms step_avg:152.15ms
step:363/1480 train_time:53706ms step_avg:152.14ms
step:364/1480 train_time:53858ms step_avg:152.14ms
step:365/1480 train_time:54008ms step_avg:152.14ms
step:366/1480 train_time:54160ms step_avg:152.14ms
step:367/1480 train_time:54310ms step_avg:152.13ms
step:368/1480 train_time:54462ms step_avg:152.13ms
step:369/1480 train_time:54611ms step_avg:152.12ms
step:370/1480 train_time:54762ms step_avg:152.12ms
step:371/1480 train_time:54913ms step_avg:152.11ms
step:372/1480 train_time:55064ms step_avg:152.11ms
step:373/1480 train_time:55215ms step_avg:152.11ms
step:374/1480 train_time:55365ms step_avg:152.10ms
step:375/1480 train_time:55517ms step_avg:152.10ms
step:375/1480 val_loss:3.8024 train_time:55584ms step_avg:152.28ms
step:376/1480 train_time:55679ms step_avg:152.13ms
step:377/1480 train_time:55825ms step_avg:152.11ms
step:378/1480 train_time:55975ms step_avg:152.11ms
step:379/1480 train_time:56142ms step_avg:152.15ms
step:380/1480 train_time:56276ms step_avg:152.10ms
step:381/1480 train_time:56426ms step_avg:152.09ms
step:382/1480 train_time:56577ms step_avg:152.09ms
step:383/1480 train_time:56729ms step_avg:152.09ms
step:384/1480 train_time:56881ms step_avg:152.09ms
step:385/1480 train_time:57031ms step_avg:152.08ms
step:386/1480 train_time:57182ms step_avg:152.08ms
step:387/1480 train_time:57333ms step_avg:152.08ms
step:388/1480 train_time:57483ms step_avg:152.07ms
step:389/1480 train_time:57634ms step_avg:152.07ms
step:390/1480 train_time:57785ms step_avg:152.06ms
step:391/1480 train_time:57936ms step_avg:152.06ms
step:392/1480 train_time:58086ms step_avg:152.06ms
step:393/1480 train_time:58238ms step_avg:152.06ms
step:394/1480 train_time:58387ms step_avg:152.05ms
step:395/1480 train_time:58539ms step_avg:152.05ms
step:396/1480 train_time:58689ms step_avg:152.04ms
step:397/1480 train_time:58840ms step_avg:152.04ms
step:398/1480 train_time:58991ms step_avg:152.04ms
step:399/1480 train_time:59142ms step_avg:152.04ms
step:400/1480 train_time:59293ms step_avg:152.03ms
step:401/1480 train_time:59444ms step_avg:152.03ms
step:402/1480 train_time:59596ms step_avg:152.03ms
step:403/1480 train_time:59745ms step_avg:152.02ms
step:404/1480 train_time:59897ms step_avg:152.02ms
step:405/1480 train_time:60047ms step_avg:152.02ms
step:406/1480 train_time:60199ms step_avg:152.02ms
step:407/1480 train_time:60351ms step_avg:152.02ms
step:408/1480 train_time:60502ms step_avg:152.01ms
step:409/1480 train_time:60652ms step_avg:152.01ms
step:410/1480 train_time:60803ms step_avg:152.01ms
step:411/1480 train_time:60954ms step_avg:152.01ms
step:412/1480 train_time:61104ms step_avg:152.00ms
step:413/1480 train_time:61256ms step_avg:152.00ms
step:414/1480 train_time:61407ms step_avg:152.00ms
step:415/1480 train_time:61558ms step_avg:152.00ms
step:416/1480 train_time:61708ms step_avg:151.99ms
step:417/1480 train_time:61860ms step_avg:151.99ms
step:418/1480 train_time:62010ms step_avg:151.98ms
step:419/1480 train_time:62161ms step_avg:151.98ms
step:420/1480 train_time:62311ms step_avg:151.98ms
step:421/1480 train_time:62462ms step_avg:151.97ms
step:422/1480 train_time:62612ms step_avg:151.97ms
step:423/1480 train_time:62763ms step_avg:151.97ms
step:424/1480 train_time:62914ms step_avg:151.97ms
step:425/1480 train_time:63065ms step_avg:151.96ms
step:426/1480 train_time:63216ms step_avg:151.96ms
step:427/1480 train_time:63366ms step_avg:151.96ms
step:428/1480 train_time:63517ms step_avg:151.95ms
step:429/1480 train_time:63669ms step_avg:151.95ms
step:430/1480 train_time:63820ms step_avg:151.95ms
step:431/1480 train_time:63971ms step_avg:151.95ms
step:432/1480 train_time:64122ms step_avg:151.95ms
step:433/1480 train_time:64274ms step_avg:151.95ms
step:434/1480 train_time:64425ms step_avg:151.95ms
step:435/1480 train_time:64576ms step_avg:151.94ms
step:436/1480 train_time:64726ms step_avg:151.94ms
step:437/1480 train_time:64877ms step_avg:151.94ms
step:438/1480 train_time:65027ms step_avg:151.93ms
step:439/1480 train_time:65178ms step_avg:151.93ms
step:440/1480 train_time:65328ms step_avg:151.93ms
step:441/1480 train_time:65481ms step_avg:151.93ms
step:442/1480 train_time:65634ms step_avg:151.93ms
step:443/1480 train_time:65787ms step_avg:151.93ms
step:444/1480 train_time:65940ms step_avg:151.94ms
step:445/1480 train_time:66092ms step_avg:151.94ms
step:446/1480 train_time:66244ms step_avg:151.94ms
step:447/1480 train_time:66398ms step_avg:151.94ms
step:448/1480 train_time:66552ms step_avg:151.95ms
step:449/1480 train_time:66705ms step_avg:151.95ms
step:450/1480 train_time:66859ms step_avg:151.95ms
step:451/1480 train_time:67012ms step_avg:151.95ms
step:452/1480 train_time:67164ms step_avg:151.96ms
step:453/1480 train_time:67317ms step_avg:151.96ms
step:454/1480 train_time:67471ms step_avg:151.96ms
step:455/1480 train_time:67623ms step_avg:151.96ms
step:456/1480 train_time:67777ms step_avg:151.97ms
step:457/1480 train_time:67930ms step_avg:151.97ms
step:458/1480 train_time:68083ms step_avg:151.97ms
step:459/1480 train_time:68238ms step_avg:151.98ms
step:460/1480 train_time:68389ms step_avg:151.98ms
step:461/1480 train_time:68542ms step_avg:151.98ms
step:462/1480 train_time:68696ms step_avg:151.98ms
step:463/1480 train_time:68849ms step_avg:151.98ms
step:464/1480 train_time:69002ms step_avg:151.99ms
step:465/1480 train_time:69155ms step_avg:151.99ms
step:466/1480 train_time:69308ms step_avg:151.99ms
step:467/1480 train_time:69461ms step_avg:151.99ms
step:468/1480 train_time:69614ms step_avg:152.00ms
step:469/1480 train_time:69766ms step_avg:152.00ms
step:470/1480 train_time:69919ms step_avg:152.00ms
step:471/1480 train_time:70073ms step_avg:152.00ms
step:472/1480 train_time:70226ms step_avg:152.01ms
step:473/1480 train_time:70380ms step_avg:152.01ms
step:474/1480 train_time:70532ms step_avg:152.01ms
step:475/1480 train_time:70685ms step_avg:152.01ms
step:476/1480 train_time:70839ms step_avg:152.01ms
step:477/1480 train_time:70991ms step_avg:152.02ms
step:478/1480 train_time:71145ms step_avg:152.02ms
step:479/1480 train_time:71298ms step_avg:152.02ms
step:480/1480 train_time:71450ms step_avg:152.02ms
step:481/1480 train_time:71603ms step_avg:152.02ms
step:482/1480 train_time:71757ms step_avg:152.03ms
step:483/1480 train_time:71909ms step_avg:152.03ms
step:484/1480 train_time:72062ms step_avg:152.03ms
step:485/1480 train_time:72215ms step_avg:152.03ms
step:486/1480 train_time:72368ms step_avg:152.03ms
step:487/1480 train_time:72520ms step_avg:152.03ms
step:488/1480 train_time:72672ms step_avg:152.03ms
step:489/1480 train_time:72826ms step_avg:152.04ms
step:490/1480 train_time:72980ms step_avg:152.04ms
step:491/1480 train_time:73133ms step_avg:152.04ms
step:492/1480 train_time:73285ms step_avg:152.04ms
step:493/1480 train_time:73439ms step_avg:152.05ms
step:494/1480 train_time:73591ms step_avg:152.05ms
step:495/1480 train_time:73744ms step_avg:152.05ms
step:496/1480 train_time:73898ms step_avg:152.05ms
step:497/1480 train_time:74051ms step_avg:152.06ms
step:498/1480 train_time:74203ms step_avg:152.06ms
step:499/1480 train_time:74357ms step_avg:152.06ms
step:500/1480 train_time:74509ms step_avg:152.06ms
step:500/1480 val_loss:3.6838 train_time:74579ms step_avg:152.20ms
step:501/1480 train_time:74672ms step_avg:152.08ms
step:502/1480 train_time:74821ms step_avg:152.07ms
step:503/1480 train_time:74974ms step_avg:152.08ms
step:504/1480 train_time:75126ms step_avg:152.08ms
step:505/1480 train_time:75280ms step_avg:152.08ms
step:506/1480 train_time:75431ms step_avg:152.08ms
step:507/1480 train_time:75585ms step_avg:152.08ms
step:508/1480 train_time:75739ms step_avg:152.09ms
step:509/1480 train_time:75892ms step_avg:152.09ms
step:510/1480 train_time:76045ms step_avg:152.09ms
step:511/1480 train_time:76198ms step_avg:152.09ms
step:512/1480 train_time:76350ms step_avg:152.09ms
step:513/1480 train_time:76504ms step_avg:152.10ms
step:514/1480 train_time:76657ms step_avg:152.10ms
step:515/1480 train_time:76811ms step_avg:152.10ms
step:516/1480 train_time:76965ms step_avg:152.10ms
step:517/1480 train_time:77118ms step_avg:152.11ms
step:518/1480 train_time:77270ms step_avg:152.11ms
step:519/1480 train_time:77422ms step_avg:152.11ms
step:520/1480 train_time:77577ms step_avg:152.11ms
step:521/1480 train_time:77731ms step_avg:152.12ms
step:522/1480 train_time:77884ms step_avg:152.12ms
step:523/1480 train_time:78037ms step_avg:152.12ms
step:524/1480 train_time:78190ms step_avg:152.12ms
step:525/1480 train_time:78343ms step_avg:152.12ms
step:526/1480 train_time:78495ms step_avg:152.12ms
step:527/1480 train_time:78647ms step_avg:152.12ms
step:528/1480 train_time:78801ms step_avg:152.13ms
step:529/1480 train_time:78953ms step_avg:152.13ms
step:530/1480 train_time:79107ms step_avg:152.13ms
step:531/1480 train_time:79260ms step_avg:152.13ms
step:532/1480 train_time:79412ms step_avg:152.13ms
step:533/1480 train_time:79564ms step_avg:152.13ms
step:534/1480 train_time:79716ms step_avg:152.13ms
step:535/1480 train_time:79869ms step_avg:152.13ms
step:536/1480 train_time:80022ms step_avg:152.13ms
step:537/1480 train_time:80177ms step_avg:152.14ms
step:538/1480 train_time:80330ms step_avg:152.14ms
step:539/1480 train_time:80485ms step_avg:152.15ms
step:540/1480 train_time:80638ms step_avg:152.15ms
step:541/1480 train_time:80790ms step_avg:152.15ms
step:542/1480 train_time:80943ms step_avg:152.15ms
step:543/1480 train_time:81095ms step_avg:152.15ms
step:544/1480 train_time:81247ms step_avg:152.15ms
step:545/1480 train_time:81401ms step_avg:152.15ms
step:546/1480 train_time:81555ms step_avg:152.15ms
step:547/1480 train_time:81707ms step_avg:152.15ms
step:548/1480 train_time:81861ms step_avg:152.16ms
step:549/1480 train_time:82013ms step_avg:152.16ms
step:550/1480 train_time:82166ms step_avg:152.16ms
step:551/1480 train_time:82320ms step_avg:152.16ms
step:552/1480 train_time:82476ms step_avg:152.17ms
step:553/1480 train_time:82631ms step_avg:152.18ms
step:554/1480 train_time:82786ms step_avg:152.18ms
step:555/1480 train_time:82940ms step_avg:152.18ms
step:556/1480 train_time:83095ms step_avg:152.19ms
step:557/1480 train_time:83250ms step_avg:152.19ms
step:558/1480 train_time:83406ms step_avg:152.20ms
step:559/1480 train_time:83560ms step_avg:152.20ms
step:560/1480 train_time:83714ms step_avg:152.21ms
step:561/1480 train_time:83869ms step_avg:152.21ms
step:562/1480 train_time:84023ms step_avg:152.22ms
step:563/1480 train_time:84177ms step_avg:152.22ms
step:564/1480 train_time:84332ms step_avg:152.22ms
step:565/1480 train_time:84487ms step_avg:152.23ms
step:566/1480 train_time:84641ms step_avg:152.23ms
step:567/1480 train_time:84795ms step_avg:152.24ms
step:568/1480 train_time:84949ms step_avg:152.24ms
step:569/1480 train_time:85114ms step_avg:152.26ms
step:570/1480 train_time:85259ms step_avg:152.25ms
step:571/1480 train_time:85414ms step_avg:152.25ms
step:572/1480 train_time:85568ms step_avg:152.26ms
step:573/1480 train_time:85722ms step_avg:152.26ms
step:574/1480 train_time:85879ms step_avg:152.27ms
step:575/1480 train_time:86034ms step_avg:152.27ms
step:576/1480 train_time:86188ms step_avg:152.28ms
step:577/1480 train_time:86343ms step_avg:152.28ms
step:578/1480 train_time:86497ms step_avg:152.28ms
step:579/1480 train_time:86651ms step_avg:152.29ms
step:580/1480 train_time:86806ms step_avg:152.29ms
step:581/1480 train_time:86962ms step_avg:152.30ms
step:582/1480 train_time:87117ms step_avg:152.30ms
step:583/1480 train_time:87271ms step_avg:152.30ms
step:584/1480 train_time:87426ms step_avg:152.31ms
step:585/1480 train_time:87581ms step_avg:152.31ms
step:586/1480 train_time:87735ms step_avg:152.32ms
step:587/1480 train_time:87889ms step_avg:152.32ms
step:588/1480 train_time:88045ms step_avg:152.33ms
step:589/1480 train_time:88199ms step_avg:152.33ms
step:590/1480 train_time:88354ms step_avg:152.33ms
step:591/1480 train_time:88508ms step_avg:152.34ms
step:592/1480 train_time:88664ms step_avg:152.34ms
step:593/1480 train_time:88819ms step_avg:152.35ms
step:594/1480 train_time:88974ms step_avg:152.35ms
step:595/1480 train_time:89130ms step_avg:152.36ms
step:596/1480 train_time:89287ms step_avg:152.37ms
step:597/1480 train_time:89442ms step_avg:152.37ms
step:598/1480 train_time:89596ms step_avg:152.37ms
step:599/1480 train_time:89751ms step_avg:152.38ms
step:600/1480 train_time:89907ms step_avg:152.38ms
step:601/1480 train_time:90063ms step_avg:152.39ms
step:602/1480 train_time:90217ms step_avg:152.39ms
step:603/1480 train_time:90372ms step_avg:152.40ms
step:604/1480 train_time:90526ms step_avg:152.40ms
step:605/1480 train_time:90682ms step_avg:152.41ms
step:606/1480 train_time:90838ms step_avg:152.41ms
step:607/1480 train_time:90993ms step_avg:152.42ms
step:608/1480 train_time:91147ms step_avg:152.42ms
step:609/1480 train_time:91302ms step_avg:152.42ms
step:610/1480 train_time:91456ms step_avg:152.43ms
step:611/1480 train_time:91612ms step_avg:152.43ms
step:612/1480 train_time:91766ms step_avg:152.44ms
step:613/1480 train_time:91920ms step_avg:152.44ms
step:614/1480 train_time:92076ms step_avg:152.44ms
step:615/1480 train_time:92230ms step_avg:152.45ms
step:616/1480 train_time:92384ms step_avg:152.45ms
step:617/1480 train_time:92539ms step_avg:152.45ms
step:618/1480 train_time:92693ms step_avg:152.46ms
step:619/1480 train_time:92849ms step_avg:152.46ms
step:620/1480 train_time:93004ms step_avg:152.47ms
step:621/1480 train_time:93160ms step_avg:152.47ms
step:622/1480 train_time:93314ms step_avg:152.47ms
step:623/1480 train_time:93468ms step_avg:152.48ms
step:624/1480 train_time:93624ms step_avg:152.48ms
step:625/1480 train_time:93779ms step_avg:152.49ms
step:625/1480 val_loss:3.6046 train_time:93851ms step_avg:152.60ms
step:626/1480 train_time:93945ms step_avg:152.51ms
step:627/1480 train_time:94094ms step_avg:152.50ms
step:628/1480 train_time:94248ms step_avg:152.51ms
step:629/1480 train_time:94403ms step_avg:152.51ms
step:630/1480 train_time:94558ms step_avg:152.51ms
step:631/1480 train_time:94712ms step_avg:152.52ms
step:632/1480 train_time:94867ms step_avg:152.52ms
step:633/1480 train_time:95021ms step_avg:152.52ms
step:634/1480 train_time:95179ms step_avg:152.53ms
step:635/1480 train_time:95333ms step_avg:152.53ms
step:636/1480 train_time:95487ms step_avg:152.54ms
step:637/1480 train_time:95642ms step_avg:152.54ms
step:638/1480 train_time:95797ms step_avg:152.54ms
step:639/1480 train_time:95952ms step_avg:152.55ms
step:640/1480 train_time:96107ms step_avg:152.55ms
step:641/1480 train_time:96262ms step_avg:152.55ms
step:642/1480 train_time:96415ms step_avg:152.56ms
step:643/1480 train_time:96570ms step_avg:152.56ms
step:644/1480 train_time:96724ms step_avg:152.56ms
step:645/1480 train_time:96879ms step_avg:152.57ms
step:646/1480 train_time:97034ms step_avg:152.57ms
step:647/1480 train_time:97188ms step_avg:152.57ms
step:648/1480 train_time:97344ms step_avg:152.58ms
step:649/1480 train_time:97498ms step_avg:152.58ms
step:650/1480 train_time:97653ms step_avg:152.58ms
step:651/1480 train_time:97808ms step_avg:152.59ms
step:652/1480 train_time:97964ms step_avg:152.59ms
step:653/1480 train_time:98117ms step_avg:152.59ms
step:654/1480 train_time:98272ms step_avg:152.60ms
step:655/1480 train_time:98426ms step_avg:152.60ms
step:656/1480 train_time:98583ms step_avg:152.60ms
step:657/1480 train_time:98737ms step_avg:152.61ms
step:658/1480 train_time:98892ms step_avg:152.61ms
step:659/1480 train_time:99047ms step_avg:152.61ms
step:660/1480 train_time:99203ms step_avg:152.62ms
step:661/1480 train_time:99360ms step_avg:152.63ms
step:662/1480 train_time:99516ms step_avg:152.63ms
step:663/1480 train_time:99671ms step_avg:152.64ms
step:664/1480 train_time:99828ms step_avg:152.64ms
step:665/1480 train_time:99985ms step_avg:152.65ms
step:666/1480 train_time:100141ms step_avg:152.65ms
step:667/1480 train_time:100297ms step_avg:152.66ms
step:668/1480 train_time:100453ms step_avg:152.66ms
step:669/1480 train_time:100612ms step_avg:152.67ms
step:670/1480 train_time:100767ms step_avg:152.68ms
step:671/1480 train_time:100924ms step_avg:152.68ms
step:672/1480 train_time:101082ms step_avg:152.69ms
step:673/1480 train_time:101238ms step_avg:152.70ms
step:674/1480 train_time:101394ms step_avg:152.70ms
step:675/1480 train_time:101552ms step_avg:152.71ms
step:676/1480 train_time:101709ms step_avg:152.72ms
step:677/1480 train_time:101865ms step_avg:152.72ms
step:678/1480 train_time:102021ms step_avg:152.73ms
step:679/1480 train_time:102178ms step_avg:152.73ms
step:680/1480 train_time:102334ms step_avg:152.74ms
step:681/1480 train_time:102490ms step_avg:152.74ms
step:682/1480 train_time:102647ms step_avg:152.75ms
step:683/1480 train_time:102804ms step_avg:152.75ms
step:684/1480 train_time:102959ms step_avg:152.76ms
step:685/1480 train_time:103115ms step_avg:152.76ms
step:686/1480 train_time:103272ms step_avg:152.77ms
step:687/1480 train_time:103427ms step_avg:152.77ms
step:688/1480 train_time:103585ms step_avg:152.78ms
step:689/1480 train_time:103743ms step_avg:152.79ms
step:690/1480 train_time:103900ms step_avg:152.79ms
step:691/1480 train_time:104056ms step_avg:152.80ms
step:692/1480 train_time:104212ms step_avg:152.80ms
step:693/1480 train_time:104367ms step_avg:152.81ms
step:694/1480 train_time:104523ms step_avg:152.81ms
step:695/1480 train_time:104678ms step_avg:152.81ms
step:696/1480 train_time:104835ms step_avg:152.82ms
step:697/1480 train_time:104991ms step_avg:152.83ms
step:698/1480 train_time:105147ms step_avg:152.83ms
step:699/1480 train_time:105304ms step_avg:152.84ms
step:700/1480 train_time:105460ms step_avg:152.84ms
step:701/1480 train_time:105615ms step_avg:152.84ms
step:702/1480 train_time:105772ms step_avg:152.85ms
step:703/1480 train_time:105928ms step_avg:152.85ms
step:704/1480 train_time:106084ms step_avg:152.86ms
step:705/1480 train_time:106240ms step_avg:152.86ms
step:706/1480 train_time:106399ms step_avg:152.87ms
step:707/1480 train_time:106554ms step_avg:152.88ms
step:708/1480 train_time:106712ms step_avg:152.88ms
step:709/1480 train_time:106867ms step_avg:152.89ms
step:710/1480 train_time:107022ms step_avg:152.89ms
step:711/1480 train_time:107180ms step_avg:152.90ms
step:712/1480 train_time:107339ms step_avg:152.90ms
step:713/1480 train_time:107496ms step_avg:152.91ms
step:714/1480 train_time:107653ms step_avg:152.92ms
step:715/1480 train_time:107809ms step_avg:152.92ms
step:716/1480 train_time:107965ms step_avg:152.92ms
step:717/1480 train_time:108121ms step_avg:152.93ms
step:718/1480 train_time:108277ms step_avg:152.93ms
step:719/1480 train_time:108432ms step_avg:152.94ms
step:720/1480 train_time:108591ms step_avg:152.94ms
step:721/1480 train_time:108747ms step_avg:152.95ms
step:722/1480 train_time:108904ms step_avg:152.95ms
step:723/1480 train_time:109060ms step_avg:152.96ms
step:724/1480 train_time:109216ms step_avg:152.96ms
step:725/1480 train_time:109373ms step_avg:152.97ms
step:726/1480 train_time:109529ms step_avg:152.97ms
step:727/1480 train_time:109687ms step_avg:152.98ms
step:728/1480 train_time:109843ms step_avg:152.98ms
step:729/1480 train_time:109999ms step_avg:152.99ms
step:730/1480 train_time:110157ms step_avg:153.00ms
step:731/1480 train_time:110315ms step_avg:153.00ms
step:732/1480 train_time:110471ms step_avg:153.01ms
step:733/1480 train_time:110626ms step_avg:153.01ms
step:734/1480 train_time:110784ms step_avg:153.02ms
step:735/1480 train_time:110941ms step_avg:153.02ms
step:736/1480 train_time:111097ms step_avg:153.03ms
step:737/1480 train_time:111254ms step_avg:153.03ms
step:738/1480 train_time:111410ms step_avg:153.04ms
step:739/1480 train_time:111565ms step_avg:153.04ms
step:740/1480 train_time:111723ms step_avg:153.05ms
step:741/1480 train_time:111880ms step_avg:153.05ms
step:742/1480 train_time:112036ms step_avg:153.05ms
step:743/1480 train_time:112191ms step_avg:153.06ms
step:744/1480 train_time:112348ms step_avg:153.06ms
step:745/1480 train_time:112506ms step_avg:153.07ms
step:746/1480 train_time:112661ms step_avg:153.07ms
step:747/1480 train_time:112817ms step_avg:153.08ms
step:748/1480 train_time:112977ms step_avg:153.08ms
step:749/1480 train_time:113134ms step_avg:153.09ms
step:750/1480 train_time:113289ms step_avg:153.09ms
step:750/1480 val_loss:3.5500 train_time:113361ms step_avg:153.19ms
step:751/1480 train_time:113452ms step_avg:153.11ms
step:752/1480 train_time:113607ms step_avg:153.11ms
step:753/1480 train_time:113764ms step_avg:153.11ms
step:754/1480 train_time:113921ms step_avg:153.12ms
step:755/1480 train_time:114077ms step_avg:153.12ms
step:756/1480 train_time:114233ms step_avg:153.13ms
step:757/1480 train_time:114391ms step_avg:153.13ms
step:758/1480 train_time:114547ms step_avg:153.14ms
step:759/1480 train_time:114713ms step_avg:153.15ms
step:760/1480 train_time:114861ms step_avg:153.15ms
step:761/1480 train_time:115017ms step_avg:153.15ms
step:762/1480 train_time:115174ms step_avg:153.16ms
step:763/1480 train_time:115331ms step_avg:153.16ms
step:764/1480 train_time:115488ms step_avg:153.17ms
step:765/1480 train_time:115645ms step_avg:153.17ms
step:766/1480 train_time:115804ms step_avg:153.18ms
step:767/1480 train_time:115961ms step_avg:153.19ms
step:768/1480 train_time:116118ms step_avg:153.19ms
step:769/1480 train_time:116275ms step_avg:153.20ms
step:770/1480 train_time:116432ms step_avg:153.20ms
step:771/1480 train_time:116591ms step_avg:153.21ms
step:772/1480 train_time:116747ms step_avg:153.21ms
step:773/1480 train_time:116906ms step_avg:153.22ms
step:774/1480 train_time:117064ms step_avg:153.22ms
step:775/1480 train_time:117223ms step_avg:153.23ms
step:776/1480 train_time:117380ms step_avg:153.24ms
step:777/1480 train_time:117541ms step_avg:153.25ms
step:778/1480 train_time:117700ms step_avg:153.25ms
step:779/1480 train_time:117857ms step_avg:153.26ms
step:780/1480 train_time:118014ms step_avg:153.27ms
step:781/1480 train_time:118172ms step_avg:153.27ms
step:782/1480 train_time:118330ms step_avg:153.28ms
step:783/1480 train_time:118486ms step_avg:153.28ms
step:784/1480 train_time:118646ms step_avg:153.29ms
step:785/1480 train_time:118805ms step_avg:153.30ms
step:786/1480 train_time:118963ms step_avg:153.30ms
step:787/1480 train_time:119122ms step_avg:153.31ms
step:788/1480 train_time:119280ms step_avg:153.32ms
step:789/1480 train_time:119436ms step_avg:153.32ms
step:790/1480 train_time:119594ms step_avg:153.33ms
step:791/1480 train_time:119756ms step_avg:153.34ms
step:792/1480 train_time:119914ms step_avg:153.34ms
step:793/1480 train_time:120071ms step_avg:153.35ms
step:794/1480 train_time:120229ms step_avg:153.35ms
step:795/1480 train_time:120388ms step_avg:153.36ms
step:796/1480 train_time:120550ms step_avg:153.37ms
step:797/1480 train_time:120709ms step_avg:153.38ms
step:798/1480 train_time:120869ms step_avg:153.39ms
step:799/1480 train_time:121031ms step_avg:153.40ms
step:800/1480 train_time:121190ms step_avg:153.40ms
step:801/1480 train_time:121348ms step_avg:153.41ms
step:802/1480 train_time:121505ms step_avg:153.42ms
step:803/1480 train_time:121664ms step_avg:153.42ms
step:804/1480 train_time:121822ms step_avg:153.43ms
step:805/1480 train_time:121984ms step_avg:153.44ms
step:806/1480 train_time:122141ms step_avg:153.44ms
step:807/1480 train_time:122298ms step_avg:153.45ms
step:808/1480 train_time:122456ms step_avg:153.45ms
step:809/1480 train_time:122613ms step_avg:153.46ms
step:810/1480 train_time:122771ms step_avg:153.46ms
step:811/1480 train_time:122930ms step_avg:153.47ms
step:812/1480 train_time:123086ms step_avg:153.47ms
step:813/1480 train_time:123243ms step_avg:153.48ms
step:814/1480 train_time:123401ms step_avg:153.48ms
step:815/1480 train_time:123558ms step_avg:153.49ms
step:816/1480 train_time:123716ms step_avg:153.49ms
step:817/1480 train_time:123874ms step_avg:153.50ms
step:818/1480 train_time:124030ms step_avg:153.50ms
step:819/1480 train_time:124189ms step_avg:153.51ms
step:820/1480 train_time:124348ms step_avg:153.52ms
step:821/1480 train_time:124505ms step_avg:153.52ms
step:822/1480 train_time:124663ms step_avg:153.53ms
step:823/1480 train_time:124822ms step_avg:153.53ms
step:824/1480 train_time:124980ms step_avg:153.54ms
step:825/1480 train_time:125139ms step_avg:153.55ms
step:826/1480 train_time:125301ms step_avg:153.56ms
step:827/1480 train_time:125459ms step_avg:153.56ms
step:828/1480 train_time:125616ms step_avg:153.57ms
step:829/1480 train_time:125775ms step_avg:153.57ms
step:830/1480 train_time:125933ms step_avg:153.58ms
step:831/1480 train_time:126091ms step_avg:153.58ms
step:832/1480 train_time:126248ms step_avg:153.59ms
step:833/1480 train_time:126406ms step_avg:153.59ms
step:834/1480 train_time:126565ms step_avg:153.60ms
step:835/1480 train_time:126723ms step_avg:153.60ms
step:836/1480 train_time:126882ms step_avg:153.61ms
step:837/1480 train_time:127038ms step_avg:153.61ms
step:838/1480 train_time:127196ms step_avg:153.62ms
step:839/1480 train_time:127354ms step_avg:153.62ms
step:840/1480 train_time:127511ms step_avg:153.63ms
step:841/1480 train_time:127669ms step_avg:153.63ms
step:842/1480 train_time:127829ms step_avg:153.64ms
step:843/1480 train_time:127985ms step_avg:153.64ms
step:844/1480 train_time:128141ms step_avg:153.65ms
step:845/1480 train_time:128298ms step_avg:153.65ms
step:846/1480 train_time:128457ms step_avg:153.66ms
step:847/1480 train_time:128617ms step_avg:153.66ms
step:848/1480 train_time:128774ms step_avg:153.67ms
step:849/1480 train_time:128932ms step_avg:153.67ms
step:850/1480 train_time:129090ms step_avg:153.68ms
step:851/1480 train_time:129248ms step_avg:153.68ms
step:852/1480 train_time:129406ms step_avg:153.69ms
step:853/1480 train_time:129563ms step_avg:153.69ms
step:854/1480 train_time:129721ms step_avg:153.70ms
step:855/1480 train_time:129878ms step_avg:153.70ms
step:856/1480 train_time:130035ms step_avg:153.71ms
step:857/1480 train_time:130193ms step_avg:153.71ms
step:858/1480 train_time:130353ms step_avg:153.72ms
step:859/1480 train_time:130510ms step_avg:153.72ms
step:860/1480 train_time:130669ms step_avg:153.73ms
step:861/1480 train_time:130828ms step_avg:153.73ms
step:862/1480 train_time:130988ms step_avg:153.74ms
step:863/1480 train_time:131149ms step_avg:153.75ms
step:864/1480 train_time:131308ms step_avg:153.76ms
step:865/1480 train_time:131465ms step_avg:153.76ms
step:866/1480 train_time:131623ms step_avg:153.77ms
step:867/1480 train_time:131781ms step_avg:153.77ms
step:868/1480 train_time:131939ms step_avg:153.78ms
step:869/1480 train_time:132097ms step_avg:153.78ms
step:870/1480 train_time:132256ms step_avg:153.79ms
step:871/1480 train_time:132413ms step_avg:153.79ms
step:872/1480 train_time:132572ms step_avg:153.80ms
step:873/1480 train_time:132729ms step_avg:153.80ms
step:874/1480 train_time:132888ms step_avg:153.81ms
step:875/1480 train_time:133049ms step_avg:153.81ms
step:875/1480 val_loss:3.5043 train_time:133121ms step_avg:153.90ms
step:876/1480 train_time:133210ms step_avg:153.82ms
step:877/1480 train_time:133369ms step_avg:153.83ms
step:878/1480 train_time:133528ms step_avg:153.83ms
step:879/1480 train_time:133687ms step_avg:153.84ms
step:880/1480 train_time:133845ms step_avg:153.84ms
step:881/1480 train_time:134003ms step_avg:153.85ms
step:882/1480 train_time:134161ms step_avg:153.85ms
step:883/1480 train_time:134321ms step_avg:153.86ms
step:884/1480 train_time:134481ms step_avg:153.87ms
step:885/1480 train_time:134640ms step_avg:153.87ms
step:886/1480 train_time:134800ms step_avg:153.88ms
step:887/1480 train_time:134960ms step_avg:153.89ms
step:888/1480 train_time:135122ms step_avg:153.90ms
step:889/1480 train_time:135284ms step_avg:153.91ms
step:890/1480 train_time:135440ms step_avg:153.91ms
step:891/1480 train_time:135598ms step_avg:153.91ms
step:892/1480 train_time:135758ms step_avg:153.92ms
step:893/1480 train_time:135916ms step_avg:153.92ms
step:894/1480 train_time:136075ms step_avg:153.93ms
step:895/1480 train_time:136236ms step_avg:153.94ms
step:896/1480 train_time:136393ms step_avg:153.94ms
step:897/1480 train_time:136555ms step_avg:153.95ms
step:898/1480 train_time:136715ms step_avg:153.96ms
step:899/1480 train_time:136874ms step_avg:153.96ms
step:900/1480 train_time:137032ms step_avg:153.97ms
step:901/1480 train_time:137192ms step_avg:153.98ms
step:902/1480 train_time:137351ms step_avg:153.98ms
step:903/1480 train_time:137512ms step_avg:153.99ms
step:904/1480 train_time:137671ms step_avg:153.99ms
step:905/1480 train_time:137830ms step_avg:154.00ms
step:906/1480 train_time:137990ms step_avg:154.01ms
step:907/1480 train_time:138152ms step_avg:154.02ms
step:908/1480 train_time:138311ms step_avg:154.02ms
step:909/1480 train_time:138473ms step_avg:154.03ms
step:910/1480 train_time:138636ms step_avg:154.04ms
step:911/1480 train_time:138794ms step_avg:154.04ms
step:912/1480 train_time:138952ms step_avg:154.05ms
step:913/1480 train_time:139113ms step_avg:154.06ms
step:914/1480 train_time:139273ms step_avg:154.06ms
step:915/1480 train_time:139435ms step_avg:154.07ms
step:916/1480 train_time:139593ms step_avg:154.08ms
step:917/1480 train_time:139751ms step_avg:154.08ms
step:918/1480 train_time:139913ms step_avg:154.09ms
step:919/1480 train_time:140075ms step_avg:154.10ms
step:920/1480 train_time:140234ms step_avg:154.10ms
step:921/1480 train_time:140392ms step_avg:154.11ms
step:922/1480 train_time:140553ms step_avg:154.11ms
step:923/1480 train_time:140711ms step_avg:154.12ms
step:924/1480 train_time:140870ms step_avg:154.12ms
step:925/1480 train_time:141030ms step_avg:154.13ms
step:926/1480 train_time:141191ms step_avg:154.14ms
step:927/1480 train_time:141350ms step_avg:154.14ms
step:928/1480 train_time:141510ms step_avg:154.15ms
step:929/1480 train_time:141670ms step_avg:154.16ms
step:930/1480 train_time:141830ms step_avg:154.16ms
step:931/1480 train_time:141989ms step_avg:154.17ms
step:932/1480 train_time:142148ms step_avg:154.17ms
step:933/1480 train_time:142308ms step_avg:154.18ms
step:934/1480 train_time:142466ms step_avg:154.18ms
step:935/1480 train_time:142628ms step_avg:154.19ms
step:936/1480 train_time:142788ms step_avg:154.20ms
step:937/1480 train_time:142950ms step_avg:154.21ms
step:938/1480 train_time:143108ms step_avg:154.21ms
step:939/1480 train_time:143270ms step_avg:154.22ms
step:940/1480 train_time:143432ms step_avg:154.23ms
step:941/1480 train_time:143590ms step_avg:154.23ms
step:942/1480 train_time:143749ms step_avg:154.24ms
step:943/1480 train_time:143910ms step_avg:154.24ms
step:944/1480 train_time:144073ms step_avg:154.25ms
step:945/1480 train_time:144231ms step_avg:154.26ms
step:946/1480 train_time:144393ms step_avg:154.27ms
step:947/1480 train_time:144555ms step_avg:154.27ms
step:948/1480 train_time:144714ms step_avg:154.28ms
step:949/1480 train_time:144881ms step_avg:154.29ms
step:950/1480 train_time:145033ms step_avg:154.29ms
step:951/1480 train_time:145194ms step_avg:154.30ms
step:952/1480 train_time:145353ms step_avg:154.30ms
step:953/1480 train_time:145513ms step_avg:154.31ms
step:954/1480 train_time:145675ms step_avg:154.32ms
step:955/1480 train_time:145834ms step_avg:154.32ms
step:956/1480 train_time:145993ms step_avg:154.33ms
step:957/1480 train_time:146155ms step_avg:154.34ms
step:958/1480 train_time:146318ms step_avg:154.34ms
step:959/1480 train_time:146476ms step_avg:154.35ms
step:960/1480 train_time:146636ms step_avg:154.35ms
step:961/1480 train_time:146794ms step_avg:154.36ms
step:962/1480 train_time:146955ms step_avg:154.36ms
step:963/1480 train_time:147116ms step_avg:154.37ms
step:964/1480 train_time:147277ms step_avg:154.38ms
step:965/1480 train_time:147436ms step_avg:154.38ms
step:966/1480 train_time:147593ms step_avg:154.39ms
step:967/1480 train_time:147753ms step_avg:154.39ms
step:968/1480 train_time:147912ms step_avg:154.40ms
step:969/1480 train_time:148073ms step_avg:154.40ms
step:970/1480 train_time:148231ms step_avg:154.41ms
step:971/1480 train_time:148391ms step_avg:154.41ms
step:972/1480 train_time:148549ms step_avg:154.42ms
step:973/1480 train_time:148709ms step_avg:154.42ms
step:974/1480 train_time:148868ms step_avg:154.43ms
step:975/1480 train_time:149029ms step_avg:154.43ms
step:976/1480 train_time:149189ms step_avg:154.44ms
step:977/1480 train_time:149348ms step_avg:154.44ms
step:978/1480 train_time:149508ms step_avg:154.45ms
step:979/1480 train_time:149668ms step_avg:154.46ms
step:980/1480 train_time:149829ms step_avg:154.46ms
step:981/1480 train_time:149989ms step_avg:154.47ms
step:982/1480 train_time:150149ms step_avg:154.47ms
step:983/1480 train_time:150310ms step_avg:154.48ms
step:984/1480 train_time:150469ms step_avg:154.49ms
step:985/1480 train_time:150630ms step_avg:154.49ms
step:986/1480 train_time:150791ms step_avg:154.50ms
step:987/1480 train_time:150950ms step_avg:154.50ms
step:988/1480 train_time:151109ms step_avg:154.51ms
step:989/1480 train_time:151268ms step_avg:154.51ms
step:990/1480 train_time:151431ms step_avg:154.52ms
step:991/1480 train_time:151592ms step_avg:154.53ms
step:992/1480 train_time:151757ms step_avg:154.54ms
step:993/1480 train_time:151924ms step_avg:154.55ms
step:994/1480 train_time:152084ms step_avg:154.56ms
step:995/1480 train_time:152243ms step_avg:154.56ms
step:996/1480 train_time:152400ms step_avg:154.56ms
step:997/1480 train_time:152561ms step_avg:154.57ms
step:998/1480 train_time:152719ms step_avg:154.57ms
step:999/1480 train_time:152881ms step_avg:154.58ms
step:1000/1480 train_time:153044ms step_avg:154.59ms
step:1000/1480 val_loss:3.4392 train_time:153117ms step_avg:154.66ms
step:1001/1480 train_time:153207ms step_avg:154.60ms
step:1002/1480 train_time:153364ms step_avg:154.60ms
step:1003/1480 train_time:153526ms step_avg:154.61ms
step:1004/1480 train_time:153688ms step_avg:154.62ms
step:1005/1480 train_time:153847ms step_avg:154.62ms
step:1006/1480 train_time:154009ms step_avg:154.63ms
step:1007/1480 train_time:154170ms step_avg:154.63ms
step:1008/1480 train_time:154330ms step_avg:154.64ms
step:1009/1480 train_time:154497ms step_avg:154.65ms
step:1010/1480 train_time:154658ms step_avg:154.66ms
step:1011/1480 train_time:154817ms step_avg:154.66ms
step:1012/1480 train_time:154977ms step_avg:154.67ms
step:1013/1480 train_time:155137ms step_avg:154.67ms
step:1014/1480 train_time:155299ms step_avg:154.68ms
step:1015/1480 train_time:155462ms step_avg:154.69ms
step:1016/1480 train_time:155623ms step_avg:154.69ms
step:1017/1480 train_time:155784ms step_avg:154.70ms
step:1018/1480 train_time:155945ms step_avg:154.71ms
step:1019/1480 train_time:156106ms step_avg:154.71ms
step:1020/1480 train_time:156266ms step_avg:154.72ms
step:1021/1480 train_time:156425ms step_avg:154.72ms
step:1022/1480 train_time:156584ms step_avg:154.73ms
step:1023/1480 train_time:156744ms step_avg:154.73ms
step:1024/1480 train_time:156904ms step_avg:154.74ms
step:1025/1480 train_time:157066ms step_avg:154.74ms
step:1026/1480 train_time:157224ms step_avg:154.75ms
step:1027/1480 train_time:157383ms step_avg:154.75ms
step:1028/1480 train_time:157545ms step_avg:154.76ms
step:1029/1480 train_time:157708ms step_avg:154.77ms
step:1030/1480 train_time:157870ms step_avg:154.77ms
step:1031/1480 train_time:158028ms step_avg:154.78ms
step:1032/1480 train_time:158192ms step_avg:154.79ms
step:1033/1480 train_time:158352ms step_avg:154.79ms
step:1034/1480 train_time:158513ms step_avg:154.80ms
step:1035/1480 train_time:158674ms step_avg:154.80ms
step:1036/1480 train_time:158833ms step_avg:154.81ms
step:1037/1480 train_time:158996ms step_avg:154.82ms
step:1038/1480 train_time:159157ms step_avg:154.82ms
step:1039/1480 train_time:159319ms step_avg:154.83ms
step:1040/1480 train_time:159478ms step_avg:154.83ms
step:1041/1480 train_time:159639ms step_avg:154.84ms
step:1042/1480 train_time:159798ms step_avg:154.84ms
step:1043/1480 train_time:159957ms step_avg:154.85ms
step:1044/1480 train_time:160118ms step_avg:154.85ms
step:1045/1480 train_time:160280ms step_avg:154.86ms
step:1046/1480 train_time:160440ms step_avg:154.86ms
step:1047/1480 train_time:160602ms step_avg:154.87ms
step:1048/1480 train_time:160761ms step_avg:154.88ms
step:1049/1480 train_time:160921ms step_avg:154.88ms
step:1050/1480 train_time:161081ms step_avg:154.89ms
step:1051/1480 train_time:161243ms step_avg:154.89ms
step:1052/1480 train_time:161406ms step_avg:154.90ms
step:1053/1480 train_time:161566ms step_avg:154.90ms
step:1054/1480 train_time:161726ms step_avg:154.91ms
step:1055/1480 train_time:161885ms step_avg:154.91ms
step:1056/1480 train_time:162044ms step_avg:154.92ms
step:1057/1480 train_time:162204ms step_avg:154.92ms
step:1058/1480 train_time:162364ms step_avg:154.93ms
step:1059/1480 train_time:162526ms step_avg:154.93ms
step:1060/1480 train_time:162687ms step_avg:154.94ms
step:1061/1480 train_time:162845ms step_avg:154.94ms
step:1062/1480 train_time:163005ms step_avg:154.95ms
step:1063/1480 train_time:163163ms step_avg:154.95ms
step:1064/1480 train_time:163321ms step_avg:154.95ms
step:1065/1480 train_time:163483ms step_avg:154.96ms
step:1066/1480 train_time:163644ms step_avg:154.97ms
step:1067/1480 train_time:163806ms step_avg:154.97ms
step:1068/1480 train_time:163965ms step_avg:154.98ms
step:1069/1480 train_time:164128ms step_avg:154.98ms
step:1070/1480 train_time:164287ms step_avg:154.99ms
step:1071/1480 train_time:164453ms step_avg:155.00ms
step:1072/1480 train_time:164613ms step_avg:155.00ms
step:1073/1480 train_time:164772ms step_avg:155.01ms
step:1074/1480 train_time:164932ms step_avg:155.01ms
step:1075/1480 train_time:165094ms step_avg:155.02ms
step:1076/1480 train_time:165253ms step_avg:155.02ms
step:1077/1480 train_time:165412ms step_avg:155.03ms
step:1078/1480 train_time:165578ms step_avg:155.04ms
step:1079/1480 train_time:165743ms step_avg:155.05ms
step:1080/1480 train_time:165903ms step_avg:155.05ms
step:1081/1480 train_time:166062ms step_avg:155.05ms
step:1082/1480 train_time:166222ms step_avg:155.06ms
step:1083/1480 train_time:166382ms step_avg:155.06ms
step:1084/1480 train_time:166542ms step_avg:155.07ms
step:1085/1480 train_time:166703ms step_avg:155.07ms
step:1086/1480 train_time:166863ms step_avg:155.08ms
step:1087/1480 train_time:167025ms step_avg:155.08ms
step:1088/1480 train_time:167186ms step_avg:155.09ms
step:1089/1480 train_time:167351ms step_avg:155.10ms
step:1090/1480 train_time:167515ms step_avg:155.11ms
step:1091/1480 train_time:167676ms step_avg:155.11ms
step:1092/1480 train_time:167835ms step_avg:155.12ms
step:1093/1480 train_time:167997ms step_avg:155.12ms
step:1094/1480 train_time:168159ms step_avg:155.13ms
step:1095/1480 train_time:168319ms step_avg:155.13ms
step:1096/1480 train_time:168481ms step_avg:155.14ms
step:1097/1480 train_time:168644ms step_avg:155.15ms
step:1098/1480 train_time:168806ms step_avg:155.15ms
step:1099/1480 train_time:168967ms step_avg:155.16ms
step:1100/1480 train_time:169133ms step_avg:155.17ms
step:1101/1480 train_time:169297ms step_avg:155.18ms
step:1102/1480 train_time:169459ms step_avg:155.18ms
step:1103/1480 train_time:169624ms step_avg:155.19ms
step:1104/1480 train_time:169785ms step_avg:155.20ms
step:1105/1480 train_time:169946ms step_avg:155.20ms
step:1106/1480 train_time:170107ms step_avg:155.21ms
step:1107/1480 train_time:170269ms step_avg:155.21ms
step:1108/1480 train_time:170428ms step_avg:155.22ms
step:1109/1480 train_time:170590ms step_avg:155.22ms
step:1110/1480 train_time:170751ms step_avg:155.23ms
step:1111/1480 train_time:170912ms step_avg:155.23ms
step:1112/1480 train_time:171074ms step_avg:155.24ms
step:1113/1480 train_time:171242ms step_avg:155.25ms
step:1114/1480 train_time:171405ms step_avg:155.26ms
step:1115/1480 train_time:171565ms step_avg:155.26ms
step:1116/1480 train_time:171726ms step_avg:155.27ms
step:1117/1480 train_time:171888ms step_avg:155.27ms
step:1118/1480 train_time:172054ms step_avg:155.28ms
step:1119/1480 train_time:172216ms step_avg:155.29ms
step:1120/1480 train_time:172377ms step_avg:155.29ms
step:1121/1480 train_time:172540ms step_avg:155.30ms
step:1122/1480 train_time:172700ms step_avg:155.31ms
step:1123/1480 train_time:172860ms step_avg:155.31ms
step:1124/1480 train_time:173023ms step_avg:155.32ms
step:1125/1480 train_time:173185ms step_avg:155.32ms
step:1125/1480 val_loss:3.3856 train_time:173259ms step_avg:155.39ms
step:1126/1480 train_time:173350ms step_avg:155.33ms
step:1127/1480 train_time:173513ms step_avg:155.34ms
step:1128/1480 train_time:173673ms step_avg:155.34ms
step:1129/1480 train_time:173838ms step_avg:155.35ms
step:1130/1480 train_time:173999ms step_avg:155.36ms
step:1131/1480 train_time:174166ms step_avg:155.37ms
step:1132/1480 train_time:174325ms step_avg:155.37ms
step:1133/1480 train_time:174489ms step_avg:155.38ms
step:1134/1480 train_time:174650ms step_avg:155.38ms
step:1135/1480 train_time:174812ms step_avg:155.39ms
step:1136/1480 train_time:174977ms step_avg:155.40ms
step:1137/1480 train_time:175138ms step_avg:155.40ms
step:1138/1480 train_time:175304ms step_avg:155.41ms
step:1139/1480 train_time:175473ms step_avg:155.42ms
step:1140/1480 train_time:175625ms step_avg:155.42ms
step:1141/1480 train_time:175789ms step_avg:155.43ms
step:1142/1480 train_time:175950ms step_avg:155.43ms
step:1143/1480 train_time:176114ms step_avg:155.44ms
step:1144/1480 train_time:176277ms step_avg:155.45ms
step:1145/1480 train_time:176438ms step_avg:155.45ms
step:1146/1480 train_time:176604ms step_avg:155.46ms
step:1147/1480 train_time:176765ms step_avg:155.47ms
step:1148/1480 train_time:176925ms step_avg:155.47ms
step:1149/1480 train_time:177088ms step_avg:155.48ms
step:1150/1480 train_time:177247ms step_avg:155.48ms
step:1151/1480 train_time:177412ms step_avg:155.49ms
step:1152/1480 train_time:177576ms step_avg:155.50ms
step:1153/1480 train_time:177742ms step_avg:155.51ms
step:1154/1480 train_time:177903ms step_avg:155.51ms
step:1155/1480 train_time:178064ms step_avg:155.51ms
step:1156/1480 train_time:178231ms step_avg:155.52ms
step:1157/1480 train_time:178392ms step_avg:155.53ms
step:1158/1480 train_time:178553ms step_avg:155.53ms
step:1159/1480 train_time:178715ms step_avg:155.54ms
step:1160/1480 train_time:178874ms step_avg:155.54ms
step:1161/1480 train_time:179037ms step_avg:155.55ms
step:1162/1480 train_time:179201ms step_avg:155.56ms
step:1163/1480 train_time:179364ms step_avg:155.56ms
step:1164/1480 train_time:179526ms step_avg:155.57ms
step:1165/1480 train_time:179684ms step_avg:155.57ms
step:1166/1480 train_time:179846ms step_avg:155.58ms
step:1167/1480 train_time:180005ms step_avg:155.58ms
step:1168/1480 train_time:180167ms step_avg:155.58ms
step:1169/1480 train_time:180329ms step_avg:155.59ms
step:1170/1480 train_time:180490ms step_avg:155.59ms
step:1171/1480 train_time:180650ms step_avg:155.60ms
step:1172/1480 train_time:180811ms step_avg:155.60ms
step:1173/1480 train_time:180974ms step_avg:155.61ms
step:1174/1480 train_time:181145ms step_avg:155.62ms
step:1175/1480 train_time:181307ms step_avg:155.63ms
step:1176/1480 train_time:181470ms step_avg:155.63ms
step:1177/1480 train_time:181639ms step_avg:155.65ms
step:1178/1480 train_time:181800ms step_avg:155.65ms
step:1179/1480 train_time:181961ms step_avg:155.65ms
step:1180/1480 train_time:182127ms step_avg:155.66ms
step:1181/1480 train_time:182289ms step_avg:155.67ms
step:1182/1480 train_time:182450ms step_avg:155.67ms
step:1183/1480 train_time:182612ms step_avg:155.68ms
step:1184/1480 train_time:182774ms step_avg:155.68ms
step:1185/1480 train_time:182939ms step_avg:155.69ms
step:1186/1480 train_time:183102ms step_avg:155.70ms
step:1187/1480 train_time:183272ms step_avg:155.71ms
step:1188/1480 train_time:183431ms step_avg:155.71ms
step:1189/1480 train_time:183595ms step_avg:155.72ms
step:1190/1480 train_time:183758ms step_avg:155.73ms
step:1191/1480 train_time:183921ms step_avg:155.73ms
step:1192/1480 train_time:184081ms step_avg:155.74ms
step:1193/1480 train_time:184242ms step_avg:155.74ms
step:1194/1480 train_time:184404ms step_avg:155.75ms
step:1195/1480 train_time:184566ms step_avg:155.75ms
step:1196/1480 train_time:184736ms step_avg:155.76ms
step:1197/1480 train_time:184898ms step_avg:155.77ms
step:1198/1480 train_time:185066ms step_avg:155.78ms
step:1199/1480 train_time:185227ms step_avg:155.78ms
step:1200/1480 train_time:185388ms step_avg:155.79ms
step:1201/1480 train_time:185549ms step_avg:155.79ms
step:1202/1480 train_time:185718ms step_avg:155.80ms
step:1203/1480 train_time:185885ms step_avg:155.81ms
step:1204/1480 train_time:186048ms step_avg:155.82ms
step:1205/1480 train_time:186208ms step_avg:155.82ms
step:1206/1480 train_time:186370ms step_avg:155.83ms
step:1207/1480 train_time:186531ms step_avg:155.83ms
step:1208/1480 train_time:186691ms step_avg:155.84ms
step:1209/1480 train_time:186857ms step_avg:155.84ms
step:1210/1480 train_time:187023ms step_avg:155.85ms
step:1211/1480 train_time:187186ms step_avg:155.86ms
step:1212/1480 train_time:187347ms step_avg:155.86ms
step:1213/1480 train_time:187511ms step_avg:155.87ms
step:1214/1480 train_time:187678ms step_avg:155.88ms
step:1215/1480 train_time:187842ms step_avg:155.89ms
step:1216/1480 train_time:188002ms step_avg:155.89ms
step:1217/1480 train_time:188165ms step_avg:155.89ms
step:1218/1480 train_time:188328ms step_avg:155.90ms
step:1219/1480 train_time:188498ms step_avg:155.91ms
step:1220/1480 train_time:188661ms step_avg:155.92ms
step:1221/1480 train_time:188821ms step_avg:155.92ms
step:1222/1480 train_time:188981ms step_avg:155.93ms
step:1223/1480 train_time:189145ms step_avg:155.93ms
step:1224/1480 train_time:189310ms step_avg:155.94ms
step:1225/1480 train_time:189473ms step_avg:155.95ms
step:1226/1480 train_time:189639ms step_avg:155.95ms
step:1227/1480 train_time:189805ms step_avg:155.96ms
step:1228/1480 train_time:189966ms step_avg:155.97ms
step:1229/1480 train_time:190129ms step_avg:155.97ms
step:1230/1480 train_time:190297ms step_avg:155.98ms
step:1231/1480 train_time:190464ms step_avg:155.99ms
step:1232/1480 train_time:190627ms step_avg:156.00ms
step:1233/1480 train_time:190788ms step_avg:156.00ms
step:1234/1480 train_time:190949ms step_avg:156.00ms
step:1235/1480 train_time:191115ms step_avg:156.01ms
step:1236/1480 train_time:191279ms step_avg:156.02ms
step:1237/1480 train_time:191442ms step_avg:156.02ms
step:1238/1480 train_time:191612ms step_avg:156.04ms
step:1239/1480 train_time:191774ms step_avg:156.04ms
step:1240/1480 train_time:191940ms step_avg:156.05ms
step:1241/1480 train_time:192106ms step_avg:156.06ms
step:1242/1480 train_time:192267ms step_avg:156.06ms
step:1243/1480 train_time:192429ms step_avg:156.07ms
step:1244/1480 train_time:192589ms step_avg:156.07ms
step:1245/1480 train_time:192751ms step_avg:156.07ms
step:1246/1480 train_time:192915ms step_avg:156.08ms
step:1247/1480 train_time:193079ms step_avg:156.09ms
step:1248/1480 train_time:193240ms step_avg:156.09ms
step:1249/1480 train_time:193402ms step_avg:156.09ms
step:1250/1480 train_time:193563ms step_avg:156.10ms
step:1250/1480 val_loss:3.3360 train_time:193638ms step_avg:156.16ms
step:1251/1480 train_time:193734ms step_avg:156.11ms
step:1252/1480 train_time:193899ms step_avg:156.12ms
step:1253/1480 train_time:194060ms step_avg:156.12ms
step:1254/1480 train_time:194221ms step_avg:156.13ms
step:1255/1480 train_time:194391ms step_avg:156.14ms
step:1256/1480 train_time:194555ms step_avg:156.14ms
step:1257/1480 train_time:194717ms step_avg:156.15ms
step:1258/1480 train_time:194882ms step_avg:156.16ms
step:1259/1480 train_time:195044ms step_avg:156.16ms
step:1260/1480 train_time:195203ms step_avg:156.16ms
step:1261/1480 train_time:195365ms step_avg:156.17ms
step:1262/1480 train_time:195532ms step_avg:156.18ms
step:1263/1480 train_time:195697ms step_avg:156.18ms
step:1264/1480 train_time:195857ms step_avg:156.19ms
step:1265/1480 train_time:196016ms step_avg:156.19ms
step:1266/1480 train_time:196180ms step_avg:156.19ms
step:1267/1480 train_time:196341ms step_avg:156.20ms
step:1268/1480 train_time:196503ms step_avg:156.20ms
step:1269/1480 train_time:196670ms step_avg:156.21ms
step:1270/1480 train_time:196834ms step_avg:156.22ms
step:1271/1480 train_time:196996ms step_avg:156.22ms
step:1272/1480 train_time:197157ms step_avg:156.23ms
step:1273/1480 train_time:197321ms step_avg:156.23ms
step:1274/1480 train_time:197485ms step_avg:156.24ms
step:1275/1480 train_time:197646ms step_avg:156.24ms
step:1276/1480 train_time:197806ms step_avg:156.24ms
step:1277/1480 train_time:197969ms step_avg:156.25ms
step:1278/1480 train_time:198130ms step_avg:156.25ms
step:1279/1480 train_time:198292ms step_avg:156.26ms
step:1280/1480 train_time:198460ms step_avg:156.27ms
step:1281/1480 train_time:198621ms step_avg:156.27ms
step:1282/1480 train_time:198780ms step_avg:156.27ms
step:1283/1480 train_time:198943ms step_avg:156.28ms
step:1284/1480 train_time:199105ms step_avg:156.28ms
step:1285/1480 train_time:199266ms step_avg:156.29ms
step:1286/1480 train_time:199430ms step_avg:156.29ms
step:1287/1480 train_time:199593ms step_avg:156.30ms
step:1288/1480 train_time:199755ms step_avg:156.30ms
step:1289/1480 train_time:199922ms step_avg:156.31ms
step:1290/1480 train_time:200090ms step_avg:156.32ms
step:1291/1480 train_time:200255ms step_avg:156.33ms
step:1292/1480 train_time:200418ms step_avg:156.33ms
step:1293/1480 train_time:200583ms step_avg:156.34ms
step:1294/1480 train_time:200746ms step_avg:156.34ms
step:1295/1480 train_time:200909ms step_avg:156.35ms
step:1296/1480 train_time:201072ms step_avg:156.35ms
step:1297/1480 train_time:201236ms step_avg:156.36ms
step:1298/1480 train_time:201398ms step_avg:156.37ms
step:1299/1480 train_time:201561ms step_avg:156.37ms
step:1300/1480 train_time:201722ms step_avg:156.37ms
step:1301/1480 train_time:201881ms step_avg:156.38ms
step:1302/1480 train_time:202047ms step_avg:156.38ms
step:1303/1480 train_time:202218ms step_avg:156.39ms
step:1304/1480 train_time:202382ms step_avg:156.40ms
step:1305/1480 train_time:202543ms step_avg:156.40ms
step:1306/1480 train_time:202708ms step_avg:156.41ms
step:1307/1480 train_time:202872ms step_avg:156.42ms
step:1308/1480 train_time:203034ms step_avg:156.42ms
step:1309/1480 train_time:203199ms step_avg:156.43ms
step:1310/1480 train_time:203361ms step_avg:156.43ms
step:1311/1480 train_time:203521ms step_avg:156.43ms
step:1312/1480 train_time:203687ms step_avg:156.44ms
step:1313/1480 train_time:203851ms step_avg:156.45ms
step:1314/1480 train_time:204016ms step_avg:156.45ms
step:1315/1480 train_time:204180ms step_avg:156.46ms
step:1316/1480 train_time:204339ms step_avg:156.46ms
step:1317/1480 train_time:204501ms step_avg:156.47ms
step:1318/1480 train_time:204666ms step_avg:156.47ms
step:1319/1480 train_time:204833ms step_avg:156.48ms
step:1320/1480 train_time:205001ms step_avg:156.49ms
step:1321/1480 train_time:205163ms step_avg:156.49ms
step:1322/1480 train_time:205335ms step_avg:156.51ms
step:1323/1480 train_time:205499ms step_avg:156.51ms
step:1324/1480 train_time:205660ms step_avg:156.51ms
step:1325/1480 train_time:205831ms step_avg:156.53ms
step:1326/1480 train_time:205997ms step_avg:156.53ms
step:1327/1480 train_time:206160ms step_avg:156.54ms
step:1328/1480 train_time:206323ms step_avg:156.54ms
step:1329/1480 train_time:206507ms step_avg:156.56ms
step:1330/1480 train_time:206672ms step_avg:156.57ms
step:1331/1480 train_time:206836ms step_avg:156.58ms
step:1332/1480 train_time:206998ms step_avg:156.58ms
step:1333/1480 train_time:207162ms step_avg:156.59ms
step:1334/1480 train_time:207325ms step_avg:156.59ms
step:1335/1480 train_time:207487ms step_avg:156.59ms
step:1336/1480 train_time:207657ms step_avg:156.60ms
step:1337/1480 train_time:207822ms step_avg:156.61ms
step:1338/1480 train_time:207985ms step_avg:156.62ms
step:1339/1480 train_time:208151ms step_avg:156.62ms
step:1340/1480 train_time:208316ms step_avg:156.63ms
step:1341/1480 train_time:208477ms step_avg:156.63ms
step:1342/1480 train_time:208644ms step_avg:156.64ms
step:1343/1480 train_time:208804ms step_avg:156.64ms
step:1344/1480 train_time:208966ms step_avg:156.65ms
step:1345/1480 train_time:209136ms step_avg:156.66ms
step:1346/1480 train_time:209297ms step_avg:156.66ms
step:1347/1480 train_time:209459ms step_avg:156.66ms
step:1348/1480 train_time:209623ms step_avg:156.67ms
step:1349/1480 train_time:209784ms step_avg:156.67ms
step:1350/1480 train_time:209951ms step_avg:156.68ms
step:1351/1480 train_time:210114ms step_avg:156.68ms
step:1352/1480 train_time:210277ms step_avg:156.69ms
step:1353/1480 train_time:210442ms step_avg:156.70ms
step:1354/1480 train_time:210605ms step_avg:156.70ms
step:1355/1480 train_time:210767ms step_avg:156.70ms
step:1356/1480 train_time:210930ms step_avg:156.71ms
step:1357/1480 train_time:211095ms step_avg:156.71ms
step:1358/1480 train_time:211258ms step_avg:156.72ms
step:1359/1480 train_time:211422ms step_avg:156.73ms
step:1360/1480 train_time:211589ms step_avg:156.73ms
step:1361/1480 train_time:211757ms step_avg:156.74ms
step:1362/1480 train_time:211922ms step_avg:156.75ms
step:1363/1480 train_time:212090ms step_avg:156.76ms
step:1364/1480 train_time:212253ms step_avg:156.76ms
step:1365/1480 train_time:212414ms step_avg:156.76ms
step:1366/1480 train_time:212577ms step_avg:156.77ms
step:1367/1480 train_time:212740ms step_avg:156.77ms
step:1368/1480 train_time:212904ms step_avg:156.78ms
step:1369/1480 train_time:213076ms step_avg:156.79ms
step:1370/1480 train_time:213242ms step_avg:156.80ms
step:1371/1480 train_time:213405ms step_avg:156.80ms
step:1372/1480 train_time:213574ms step_avg:156.81ms
step:1373/1480 train_time:213736ms step_avg:156.81ms
step:1374/1480 train_time:213902ms step_avg:156.82ms
step:1375/1480 train_time:214064ms step_avg:156.82ms
step:1375/1480 val_loss:3.2982 train_time:214138ms step_avg:156.88ms
step:1376/1480 train_time:214229ms step_avg:156.83ms
step:1377/1480 train_time:214393ms step_avg:156.83ms
step:1378/1480 train_time:214554ms step_avg:156.84ms
step:1379/1480 train_time:214720ms step_avg:156.84ms
step:1380/1480 train_time:214885ms step_avg:156.85ms
step:1381/1480 train_time:215052ms step_avg:156.86ms
step:1382/1480 train_time:215214ms step_avg:156.86ms
step:1383/1480 train_time:215377ms step_avg:156.87ms
step:1384/1480 train_time:215545ms step_avg:156.87ms
step:1385/1480 train_time:215706ms step_avg:156.88ms
step:1386/1480 train_time:215870ms step_avg:156.88ms
step:1387/1480 train_time:216034ms step_avg:156.89ms
step:1388/1480 train_time:216194ms step_avg:156.89ms
step:1389/1480 train_time:216361ms step_avg:156.90ms
step:1390/1480 train_time:216524ms step_avg:156.90ms
step:1391/1480 train_time:216687ms step_avg:156.91ms
step:1392/1480 train_time:216851ms step_avg:156.91ms
step:1393/1480 train_time:217012ms step_avg:156.91ms
step:1394/1480 train_time:217174ms step_avg:156.92ms
step:1395/1480 train_time:217336ms step_avg:156.92ms
step:1396/1480 train_time:217497ms step_avg:156.92ms
step:1397/1480 train_time:217659ms step_avg:156.93ms
step:1398/1480 train_time:217820ms step_avg:156.93ms
step:1399/1480 train_time:217982ms step_avg:156.93ms
step:1400/1480 train_time:218152ms step_avg:156.94ms
step:1401/1480 train_time:218312ms step_avg:156.95ms
step:1402/1480 train_time:218473ms step_avg:156.95ms
step:1403/1480 train_time:218640ms step_avg:156.96ms
step:1404/1480 train_time:218803ms step_avg:156.96ms
step:1405/1480 train_time:218968ms step_avg:156.97ms
step:1406/1480 train_time:219131ms step_avg:156.97ms
step:1407/1480 train_time:219293ms step_avg:156.97ms
step:1408/1480 train_time:219453ms step_avg:156.98ms
step:1409/1480 train_time:219626ms step_avg:156.99ms
step:1410/1480 train_time:219789ms step_avg:156.99ms
step:1411/1480 train_time:219950ms step_avg:156.99ms
step:1412/1480 train_time:220111ms step_avg:157.00ms
step:1413/1480 train_time:220273ms step_avg:157.00ms
step:1414/1480 train_time:220436ms step_avg:157.01ms
step:1415/1480 train_time:220602ms step_avg:157.01ms
step:1416/1480 train_time:220777ms step_avg:157.02ms
step:1417/1480 train_time:220941ms step_avg:157.03ms
step:1418/1480 train_time:221106ms step_avg:157.04ms
step:1419/1480 train_time:221272ms step_avg:157.04ms
step:1420/1480 train_time:221436ms step_avg:157.05ms
step:1421/1480 train_time:221602ms step_avg:157.05ms
step:1422/1480 train_time:221768ms step_avg:157.06ms
step:1423/1480 train_time:221930ms step_avg:157.06ms
step:1424/1480 train_time:222096ms step_avg:157.07ms
step:1425/1480 train_time:222267ms step_avg:157.08ms
step:1426/1480 train_time:222432ms step_avg:157.08ms
step:1427/1480 train_time:222596ms step_avg:157.09ms
step:1428/1480 train_time:222758ms step_avg:157.09ms
step:1429/1480 train_time:222919ms step_avg:157.10ms
step:1430/1480 train_time:223085ms step_avg:157.10ms
step:1431/1480 train_time:223252ms step_avg:157.11ms
step:1432/1480 train_time:223419ms step_avg:157.12ms
step:1433/1480 train_time:223589ms step_avg:157.12ms
step:1434/1480 train_time:223756ms step_avg:157.13ms
step:1435/1480 train_time:223923ms step_avg:157.14ms
step:1436/1480 train_time:224088ms step_avg:157.14ms
step:1437/1480 train_time:224250ms step_avg:157.15ms
step:1438/1480 train_time:224412ms step_avg:157.15ms
step:1439/1480 train_time:224580ms step_avg:157.16ms
step:1440/1480 train_time:224741ms step_avg:157.16ms
step:1441/1480 train_time:224907ms step_avg:157.17ms
step:1442/1480 train_time:225072ms step_avg:157.17ms
step:1443/1480 train_time:225247ms step_avg:157.19ms
step:1444/1480 train_time:225411ms step_avg:157.19ms
step:1445/1480 train_time:225574ms step_avg:157.19ms
step:1446/1480 train_time:225740ms step_avg:157.20ms
step:1447/1480 train_time:225909ms step_avg:157.21ms
step:1448/1480 train_time:226072ms step_avg:157.21ms
step:1449/1480 train_time:226234ms step_avg:157.22ms
step:1450/1480 train_time:226400ms step_avg:157.22ms
step:1451/1480 train_time:226564ms step_avg:157.23ms
step:1452/1480 train_time:226728ms step_avg:157.23ms
step:1453/1480 train_time:226892ms step_avg:157.24ms
step:1454/1480 train_time:227053ms step_avg:157.24ms
step:1455/1480 train_time:227227ms step_avg:157.25ms
step:1456/1480 train_time:227391ms step_avg:157.26ms
step:1457/1480 train_time:227553ms step_avg:157.26ms
step:1458/1480 train_time:227716ms step_avg:157.26ms
step:1459/1480 train_time:227883ms step_avg:157.27ms
step:1460/1480 train_time:228046ms step_avg:157.27ms
step:1461/1480 train_time:228211ms step_avg:157.28ms
step:1462/1480 train_time:228374ms step_avg:157.28ms
step:1463/1480 train_time:228540ms step_avg:157.29ms
step:1464/1480 train_time:228706ms step_avg:157.29ms
step:1465/1480 train_time:228870ms step_avg:157.30ms
step:1466/1480 train_time:229032ms step_avg:157.30ms
step:1467/1480 train_time:229196ms step_avg:157.31ms
step:1468/1480 train_time:229359ms step_avg:157.31ms
step:1469/1480 train_time:229524ms step_avg:157.32ms
step:1470/1480 train_time:229691ms step_avg:157.32ms
step:1471/1480 train_time:229862ms step_avg:157.33ms
step:1472/1480 train_time:230033ms step_avg:157.34ms
step:1473/1480 train_time:230195ms step_avg:157.34ms
step:1474/1480 train_time:230364ms step_avg:157.35ms
step:1475/1480 train_time:230532ms step_avg:157.36ms
step:1476/1480 train_time:230694ms step_avg:157.36ms
step:1477/1480 train_time:230863ms step_avg:157.37ms
step:1478/1480 train_time:231032ms step_avg:157.38ms
step:1479/1480 train_time:231197ms step_avg:157.38ms
step:1480/1480 train_time:231363ms step_avg:157.39ms
step:1480/1480 val_loss:3.2789 train_time:231439ms step_avg:157.44ms
peak memory consumption: 34239 MiB