a3e860f0-98ed-4845-9aee-cccd01fd9b5b.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
import contextlib
from dataclasses import dataclass
from pathlib import Path

import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.nn.attention.flex_attention import BlockMask, flex_attention #KoszarskyB

# -----------------------------------------------------------------------------
# Muon optimizer

@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert len(G.shape) == 2
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    X /= (X.norm() + eps) # ensure top singular value <= 1
    if G.size(0) > G.size(1):
        X = X.T
    for _ in range(steps):
        A = X @ X.T
        B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    if G.size(0) > G.size(1):
        X = X.T
    return X

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven't tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        ns_steps: The number of Newton-Schulz iteration steps to use.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True, ns_steps=5):
        self.world_size = int(os.environ['WORLD_SIZE'])
        self.rank = int(os.environ['RANK'])
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, ns_steps=ns_steps)
        params = list(params)
        assert all(isinstance(p, torch.Tensor) for p in params)
        sizes = {p.numel() for p in params}
        param_groups = [
            {
                'params': [p for p in params if p.numel() == size],
                'update_buffer': [
                    torch.empty(size, device='cuda', dtype=torch.bfloat16)
                    for _ in range(self.world_size)
                ],
            }
            for size in sizes
        ]
        super().__init__(param_groups, defaults)

    def step(self):

        for group in self.param_groups:

            lr = group['lr']
            momentum = group['momentum']
            nesterov = group['nesterov']
            ns_steps = group['ns_steps']
            update_buffers = group['update_buffer']
            # generate weight updates in distributed fashion
            params = group['params']
            assert len(params) % self.world_size == 0
            handle = None
            params_world = None
            def update_prev():
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffers):
                    p_world.data.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.world_size]:
                p = params[base_i + self.rank]
                g = p.grad
                assert g is not None
                state = self.state[p]
                if 'momentum_buffer' not in state:
                    state['momentum_buffer'] = torch.zeros_like(g)
                buf = state['momentum_buffer']
                buf.lerp_(g, 1 - momentum)
                g = g.lerp_(buf, momentum) if nesterov else buf
                g = zeropower_via_newtonschulz5(g, steps=ns_steps).flatten()
                update_prev()
                handle = dist.all_gather(update_buffers, g, async_op=True)
                params_world = params[base_i : base_i + self.world_size]
            update_prev()

# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):

    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=False)

    def forward(self, x):
        return F.linear(x, self.weight.to(x.dtype))

class Rotary(torch.nn.Module):

    def __init__(self, dim, base=10000):
        super().__init__()
        self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None

    def forward(self, x):
        seq_len = x.shape[1]
        if seq_len != self.seq_len_cached:
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.outer(t, self.inv_freq)
            self.seq_len_cached = seq_len
            self.cos_cached = freqs.cos()
            self.sin_cached = freqs.sin()
        cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
        # apply_rotary_emb(x, cos, sin)
        x1, x2 = x.chunk(2, dim=3)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x)

class CausalSelfAttention(nn.Module):

    def __init__(self, dim, num_heads):
        super().__init__()
        assert dim % num_heads == 0
        self.num_heads = num_heads
        self.c_q = CastedLinear(dim, dim)
        self.c_k = CastedLinear(dim, dim)
        self.c_v = CastedLinear(dim, dim)
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5]))
        self.rotary = Rotary(dim // num_heads) # dim // num_heads = head_dim
        self.c_proj = CastedLinear(dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x, vi, block_mask):
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q = self.c_q(x).view(B, T, self.num_heads, -1)
        k = self.c_k(x).view(B, T, self.num_heads, -1)
        v = self.c_v(x).view(B, T, self.num_heads, -1)
        v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @KoszarskyB & @Grad62304977
        q, k = norm(q), norm(k) # QK norm @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask, enable_gqa=True)
        y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):

    def __init__(self, dim):
        super().__init__()
        self.c_fc   = CastedLinear(dim, 4 * dim)
        self.c_proj = CastedLinear(4 * dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x):
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):

    def __init__(self, config):
        super().__init__()
        self.attn = CausalSelfAttention(config.model_dim, config.num_heads)
        self.mlp = MLP(config.model_dim)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x, vi, x0, block_mask):
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        x = x + self.attn(norm(x), vi, block_mask)
        x = x + self.mlp(norm(x))
        return x

class ValueEmbedding(nn.Module):
    def __init__(self, config: "GPTConfig"):
        super().__init__()
        self.__setattr__
        self.embed = nn.ModuleList([
            nn.Embedding(config.vocab_size, config.model_dim)
            for _ in range(6)
        ])

    def forward(self, inputs) -> "list[torch.Tensor]":
        ve = [emb(inputs) for emb in self.embed]
        ve += reversed(ve)
        return ve


# -----------------------------------------------------------------------------
# The main GPT-2 model

@dataclass
class GPTConfig:
    vocab_size : int = 50304
    num_layers : int = 12
    num_heads : int = 6 # head dim 128 suggested by @Grad62304977
    model_dim : int = 768

class GPT(nn.Module):

    def __init__(self, config: GPTConfig):
        super().__init__()
        self.num_layers = config.num_layers

        # U-net design by @brendanh0gan
        self.num_encoder_layers = config.num_layers // 2 # Half of the layers for encoder
        self.num_decoder_layers = config.num_layers - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))

        self.embed = nn.Embedding(config.vocab_size, config.model_dim)
        self.blocks = nn.ModuleList([Block(config) for _ in range(config.num_layers)])
        # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
        # U-net structure on token value embeddings by @leloykun
        self.value_embeds = ValueEmbedding(config)
        self.lm_head = CastedLinear(config.model_dim, config.vocab_size)
        self.lm_head.weight.data.zero_() # @Grad62304977

    def forward(
        self,
        inputs: torch.Tensor,
        targets: torch.Tensor,
        sliding_window_num_blocks: torch.Tensor,
    ):
        BLOCK_SIZE = 128
        assert inputs.ndim == 1
        docs = (inputs == 50256).cumsum(0)
        docs_low = docs.view(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.view(-1, BLOCK_SIZE)[:, -1].contiguous()

        def document_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            return causal_mask & document_mask

        def dense_to_ordered(dense_mask: torch.Tensor):
            num_blocks = dense_mask.sum(dim=-1, dtype=torch.int32)
            indices = dense_mask.argsort(dim=-1, descending=True, stable=True).to(torch.int32)
            return num_blocks[None, None].contiguous(), indices[None, None].contiguous()

        def create_doc_swc_block_mask(sliding_window_num_blocks: torch.Tensor):
            kv_idx = block_idx = torch.arange(512, dtype=torch.int32, device="cuda")
            q_idx = block_idx[:, None]
            causal_bm = q_idx >= kv_idx
            causal_full_bm = q_idx > kv_idx
            window_bm = q_idx - kv_idx < sliding_window_num_blocks
            window_full_bm = window_bm
            # document_bm = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
            document_bm = (docs_low[:, None] <= docs_high) & (docs_low <= docs_high[:, None])
            document_full_bm = (docs_low[:, None] == docs_high) & (docs_low == docs_high[:, None])
            nonzero_bm = causal_bm & window_bm & document_bm
            full_bm  = causal_full_bm & window_full_bm & document_full_bm
            kv_num_blocks, kv_indices = dense_to_ordered(nonzero_bm ^ full_bm)
            full_kv_num_blocks, full_kv_indices = dense_to_ordered(full_bm)
            return BlockMask.from_kv_blocks(
                kv_num_blocks,
                kv_indices,
                full_kv_num_blocks,
                full_kv_indices,
                BLOCK_SIZE=BLOCK_SIZE,
                mask_mod=document_causal,
            )

        block_mask = create_doc_swc_block_mask(sliding_window_num_blocks)

        # forward the GPT model itself
        x = self.embed(inputs[None]) # token embeddings of shape (b, t, model_dim)
        x = norm(x) # @Grad62304977
        x0 = x
        ve = self.value_embeds(inputs)
        ve_enc, ve_dec = ve[:self.num_encoder_layers], ve[self.num_encoder_layers:]

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        for i in range(self.num_encoder_layers):
            x = self.blocks[i](x, ve_enc[i], x0, block_mask)
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            # U-net structure on token value embeddings by @leloykun
            x = self.blocks[self.num_encoder_layers + i](x, ve_dec[i], x0, block_mask)

        x = norm(x)
        logits = self.lm_head(x)
        logits = 30 * torch.tanh(logits / 30) # @Grad62304977
        logits = logits.float()
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1))
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _peek_data_shard(file: Path):
    # only reads the header, returns header data
    # header is 256 int32
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    return int(header[2]) # number of tokens (claimed)

def _load_data_shard(path: Path, num_tokens):
    with path.open("rb", buffering=0) as f:
        tokens = torch.empty(num_tokens, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy())
        assert nbytes == 2 * num_tokens, "number of tokens read does not match header?"
    return tokens

class DistributedDataLoader:
    def __init__(self, filename_pattern, seq_len, process_rank, num_processes):
        self.process_rank = process_rank
        self.num_processes = num_processes
        self.seq_len = seq_len

        # glob files that match the pattern
        self.files = sorted(Path.cwd().glob(filename_pattern))
        assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"

        # load and validate all data shards, count number of tokens in total
        self.files_num_tokens = [_peek_data_shard(file) for file in self.files]
        assert min(self.files_num_tokens) >= num_processes * seq_len + 1
        self.total_num_tokens = sum(self.files_num_tokens)

        self.reset()

    def reset(self):
        self.current_shard = -1
        self.advance()

    def advance(self): # advance to next data shard
        self.current_shard = (self.current_shard + 1) % len(self.files)
        self.current_position = self.process_rank * self.seq_len
        self.tokens = _load_data_shard(self.files[self.current_shard], self.files_num_tokens[self.current_shard])

    def next_batch(self):
        batch_size = self.seq_len * self.num_processes
        buf = self.tokens[self.current_position:self.current_position+self.seq_len+1]
        # host side async is sufficient;
        # no performance improvement was observed when introducing a separate stream.
        inputs = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
        targets = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
        # advance current position and load next shard if necessary
        self.current_position += batch_size
        if self.current_position + batch_size + 1 >= len(self.tokens):
            self.advance()
        return inputs, targets

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data hyperparams
    input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
    input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
    # optimization hyperparams
    batch_size : int = 8 # batch size, in sequences, across all devices
    sequence_length : int = 64*1024 # sequence length, in tokens
    num_iterations : int = 1480 # number of iterations to run
    warmup_iters : int = 0
    cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
    weight_decay : float = 0
    # evaluation and logging hyperparams
    val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
    val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()

# set up DDP (distributed data parallel). torchrun sets this env variable
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
assert torch.cuda.is_available()
device = torch.device(f"cuda:{ddp_local_rank}")
torch.cuda.set_device(device)
print(f"using device: {device}")
dist.init_process_group(backend='nccl', device_id=device)
dist.barrier()
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = uuid.uuid4()
    logdir = Path("logs") / f"{run_id}"
    logdir.mkdir(exist_ok=True)
    logfile = Path("logs") / f"{run_id}.txt"
    print(logfile.stem)
    # create the log file
    with logfile.open("w") as f:
        # begin the log by printing this file (the Python code)
        print(code, file=f)
        print("=" * 100, file=f)
def print0(s, logonly=False):
    if master_process:
        with logfile.open("a") as f:
            if not logonly:
                print(s)
            print(s, file=f)
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running python {sys.version}")
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)

# calculate the number of steps to take in the val loop.
assert args.val_tokens % (args.sequence_length * ddp_world_size) == 0
val_steps = args.val_tokens // (args.sequence_length * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size

# load tokens
train_loader = DistributedDataLoader(args.input_bin, args.sequence_length, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, args.sequence_length, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.total_num_tokens} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.total_num_tokens} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
inputs_train, targets_train = train_loader.next_batch()

# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, num_layers=12, num_heads=6, model_dim=768))
model = model.cuda().bfloat16()
for m in model.modules():
    if isinstance(m, CastedLinear):
        m.float()
config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank], broadcast_buffers=False, gradient_as_bucket_view=True)
raw_model = model.module # always contains the "raw" unwrapped model

# init the optimizer(s)
embed_params = [*raw_model.embed.parameters(), *raw_model.value_embeds.parameters()]
optimizer1 = torch.optim.Adam(embed_params, lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.blocks.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
    assert it <= args.num_iterations
    # 1) linear warmup for warmup_iters steps
    if it < args.warmup_iters:
        return (it+1) / args.warmup_iters
    # 2) constant lr for a while
    elif it < args.num_iterations - args.cooldown_iters:
        return 1.0
    # 3) linear cooldown
    else:
        decay_ratio = (args.num_iterations - it) / args.cooldown_iters
        return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]

sliding_window_num_blocks = torch.tensor(1, dtype=torch.int32, device="cuda")
sw_num_blocks_prev = 1
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
    last_step = (step == args.num_iterations)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Linearly increase the sliding window size over training in chunks of 64 from 64 -> 1792. By @fernbear.bsky.social
    frac_done = step / args.num_iterations # training progress
    sw_num_blocks = int(((1 - frac_done) * 64 + frac_done * 1792 + 64) // 128)
    if sw_num_blocks != sw_num_blocks_prev:
        sliding_window_num_blocks.copy_(sw_num_blocks, non_blocking=True)
        sw_num_blocks_prev = sw_num_blocks

    # once in a while evaluate the validation dataset
    if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # run validation batches
        model.eval()
        val_loader.reset()
        val_loss = 0.0
        for _ in range(val_steps):
            with torch.no_grad():
                inputs_val, targets_val = val_loader.next_batch()
                val_loss += model(inputs_val, targets_val, sliding_window_num_blocks)
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        val_loss /= val_steps
        # log val loss to console and to logfile
        print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # save the state of the training process
        log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
        torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    # bit confusing: we want to make sure to eval on 0th iteration
    # but also after the very last iteration. so we loop for step <= num_iterations
    # instead of just < num_iterations (one extra due to <=), only to do
    # the validation/sampling one last time, and then we break right here as we're done.
    if last_step:
        break

    # --------------- TRAINING SECTION BEGIN -----------------
    model.train()
    for i in range(1, train_accumulation_steps + 1):
        with contextlib.ExitStack() as stack:
            if i < train_accumulation_steps: # there's no need to sync gradients every accumulation step
                stack.enter_context(model.no_sync())
            if step >= 5:
                stack.enter_context(torch.compiler.set_stance(skip_guard_eval_unsafe=True))
            model(inputs_train, targets_train, sliding_window_num_blocks).backward()
            inputs_train, targets_train = train_loader.next_batch()
    if train_accumulation_steps != 1:
        for p in model.parameters():
            p.grad /= train_accumulation_steps
    # momentum warmup for Muon
    frac = min(step/300, 1)
    for group in optimizer3.param_groups:
        group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # --------------- TRAINING SECTION END -------------------
    # everything that follows now is just diagnostics, prints, logging, etc.
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")

print0(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")

# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()

====================================================================================================
Running python 3.10.12 (main, Sep 11 2024, 15:47:36) [GCC 11.4.0]
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Wed Dec 11 08:44:46 2024       
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.183.06             Driver Version: 535.183.06   CUDA Version: 12.2     |
|-----------------------------------------+----------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
|                                         |                      |               MIG M. |
|=========================================+======================+======================|
|   0  NVIDIA H100 80GB HBM3          On  | 00000000:19:00.0 Off |                    0 |
| N/A   38C    P0             125W / 700W |   7084MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  | 00000000:3B:00.0 Off |                    0 |
| N/A   30C    P0             115W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  | 00000000:4C:00.0 Off |                    0 |
| N/A   28C    P0             111W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  | 00000000:5D:00.0 Off |                    0 |
| N/A   36C    P0             114W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  | 00000000:9B:00.0 Off |                    0 |
| N/A   38C    P0             120W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  | 00000000:BB:00.0 Off |                    0 |
| N/A   30C    P0             117W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  | 00000000:CB:00.0 Off |                    0 |
| N/A   35C    P0             119W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  | 00000000:DB:00.0 Off |                    0 |
| N/A   30C    P0             118W / 700W |   3211MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
                                                                                         
+---------------------------------------------------------------------------------------+
| Processes:                                                                            |
|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
|        ID   ID                                                             Usage      |
|=======================================================================================|
+---------------------------------------------------------------------------------------+

====================================================================================================
Training DataLoader: total number of tokens: 1000000000 across 10 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:29106ms step_avg:nanms
step:2/1480 train_time:29289ms step_avg:nanms
step:3/1480 train_time:29414ms step_avg:nanms
step:4/1480 train_time:29554ms step_avg:nanms
step:5/1480 train_time:29695ms step_avg:nanms
step:6/1480 train_time:29838ms step_avg:nanms
step:7/1480 train_time:29980ms step_avg:nanms
step:8/1480 train_time:30123ms step_avg:nanms
step:9/1480 train_time:30266ms step_avg:nanms
step:10/1480 train_time:30409ms step_avg:nanms
step:11/1480 train_time:142ms step_avg:nanms
step:12/1480 train_time:283ms step_avg:nanms
step:13/1480 train_time:426ms step_avg:141.93ms
step:14/1480 train_time:569ms step_avg:142.17ms
step:15/1480 train_time:712ms step_avg:142.38ms
step:16/1480 train_time:854ms step_avg:142.34ms
step:17/1480 train_time:996ms step_avg:142.23ms
step:18/1480 train_time:1137ms step_avg:142.14ms
step:19/1480 train_time:1281ms step_avg:142.30ms
step:20/1480 train_time:1424ms step_avg:142.44ms
step:21/1480 train_time:1568ms step_avg:142.58ms
step:22/1480 train_time:1711ms step_avg:142.60ms
step:23/1480 train_time:1853ms step_avg:142.52ms
step:24/1480 train_time:1995ms step_avg:142.50ms
step:25/1480 train_time:2136ms step_avg:142.42ms
step:26/1480 train_time:2279ms step_avg:142.43ms
step:27/1480 train_time:2420ms step_avg:142.37ms
step:28/1480 train_time:2564ms step_avg:142.44ms
step:29/1480 train_time:2710ms step_avg:142.62ms
step:30/1480 train_time:3255ms step_avg:162.77ms
step:31/1480 train_time:3353ms step_avg:159.69ms
step:32/1480 train_time:3481ms step_avg:158.23ms
step:33/1480 train_time:3624ms step_avg:157.55ms
step:34/1480 train_time:3766ms step_avg:156.91ms
step:35/1480 train_time:3908ms step_avg:156.33ms
step:36/1480 train_time:4050ms step_avg:155.78ms
step:37/1480 train_time:4193ms step_avg:155.31ms
step:38/1480 train_time:4335ms step_avg:154.84ms
step:39/1480 train_time:4477ms step_avg:154.38ms
step:40/1480 train_time:4618ms step_avg:153.94ms
step:41/1480 train_time:4762ms step_avg:153.60ms
step:42/1480 train_time:4905ms step_avg:153.29ms
step:43/1480 train_time:5048ms step_avg:152.98ms
step:44/1480 train_time:5191ms step_avg:152.67ms
step:45/1480 train_time:5333ms step_avg:152.36ms
step:46/1480 train_time:5476ms step_avg:152.11ms
step:47/1480 train_time:5617ms step_avg:151.82ms
step:48/1480 train_time:5760ms step_avg:151.59ms
step:49/1480 train_time:5904ms step_avg:151.40ms
step:50/1480 train_time:6048ms step_avg:151.21ms
step:51/1480 train_time:6193ms step_avg:151.05ms
step:52/1480 train_time:6335ms step_avg:150.83ms
step:53/1480 train_time:6478ms step_avg:150.64ms
step:54/1480 train_time:6618ms step_avg:150.42ms
step:55/1480 train_time:6761ms step_avg:150.26ms
step:56/1480 train_time:6904ms step_avg:150.09ms
step:57/1480 train_time:7050ms step_avg:149.99ms
step:58/1480 train_time:7194ms step_avg:149.88ms
step:59/1480 train_time:7336ms step_avg:149.71ms
step:60/1480 train_time:7479ms step_avg:149.59ms
step:61/1480 train_time:7622ms step_avg:149.46ms
step:62/1480 train_time:7766ms step_avg:149.34ms
step:63/1480 train_time:7909ms step_avg:149.22ms
step:64/1480 train_time:8053ms step_avg:149.13ms
step:65/1480 train_time:8194ms step_avg:148.99ms
step:66/1480 train_time:8336ms step_avg:148.85ms
step:67/1480 train_time:8478ms step_avg:148.73ms
step:68/1480 train_time:8621ms step_avg:148.64ms
step:69/1480 train_time:8763ms step_avg:148.53ms
step:70/1480 train_time:8909ms step_avg:148.49ms
step:71/1480 train_time:9052ms step_avg:148.40ms
step:72/1480 train_time:9194ms step_avg:148.30ms
step:73/1480 train_time:9335ms step_avg:148.18ms
step:74/1480 train_time:9477ms step_avg:148.08ms
step:75/1480 train_time:9619ms step_avg:147.99ms
step:76/1480 train_time:9762ms step_avg:147.91ms
step:77/1480 train_time:9906ms step_avg:147.85ms
step:78/1480 train_time:10050ms step_avg:147.80ms
step:79/1480 train_time:10587ms step_avg:153.43ms
step:80/1480 train_time:11120ms step_avg:158.86ms
step:81/1480 train_time:11220ms step_avg:158.03ms
step:82/1480 train_time:11362ms step_avg:157.81ms
step:83/1480 train_time:11504ms step_avg:157.59ms
step:84/1480 train_time:11646ms step_avg:157.37ms
step:85/1480 train_time:11790ms step_avg:157.20ms
step:86/1480 train_time:11932ms step_avg:157.00ms
step:87/1480 train_time:12076ms step_avg:156.83ms
step:88/1480 train_time:12221ms step_avg:156.67ms
step:89/1480 train_time:12364ms step_avg:156.50ms
step:90/1480 train_time:12507ms step_avg:156.34ms
step:91/1480 train_time:12650ms step_avg:156.18ms
step:92/1480 train_time:12794ms step_avg:156.02ms
step:93/1480 train_time:12936ms step_avg:155.86ms
step:94/1480 train_time:13079ms step_avg:155.70ms
step:95/1480 train_time:13223ms step_avg:155.56ms
step:96/1480 train_time:13366ms step_avg:155.42ms
step:97/1480 train_time:13909ms step_avg:159.87ms
step:98/1480 train_time:14011ms step_avg:159.22ms
step:99/1480 train_time:14154ms step_avg:159.04ms
step:100/1480 train_time:14297ms step_avg:158.86ms
step:101/1480 train_time:14445ms step_avg:158.74ms
step:102/1480 train_time:14579ms step_avg:158.47ms
step:103/1480 train_time:14723ms step_avg:158.31ms
step:104/1480 train_time:14866ms step_avg:158.15ms
step:105/1480 train_time:15011ms step_avg:158.01ms
step:106/1480 train_time:15153ms step_avg:157.84ms
step:107/1480 train_time:15297ms step_avg:157.70ms
step:108/1480 train_time:15439ms step_avg:157.54ms
step:109/1480 train_time:15580ms step_avg:157.38ms
step:110/1480 train_time:15723ms step_avg:157.23ms
step:111/1480 train_time:15870ms step_avg:157.13ms
step:112/1480 train_time:16017ms step_avg:157.03ms
step:113/1480 train_time:16161ms step_avg:156.90ms
step:114/1480 train_time:16307ms step_avg:156.80ms
step:115/1480 train_time:16453ms step_avg:156.70ms
step:116/1480 train_time:16598ms step_avg:156.58ms
step:117/1480 train_time:16744ms step_avg:156.48ms
step:118/1480 train_time:16892ms step_avg:156.40ms
step:119/1480 train_time:17036ms step_avg:156.30ms
step:120/1480 train_time:17182ms step_avg:156.20ms
step:121/1480 train_time:17328ms step_avg:156.11ms
step:122/1480 train_time:17474ms step_avg:156.02ms
step:123/1480 train_time:17618ms step_avg:155.91ms
step:124/1480 train_time:17765ms step_avg:155.83ms
step:125/1480 train_time:17912ms step_avg:155.75ms
step:125/1480 val_loss:4.4254 train_time:17976ms step_avg:156.31ms
step:126/1480 train_time:18070ms step_avg:155.77ms
step:127/1480 train_time:18212ms step_avg:155.65ms
step:128/1480 train_time:18357ms step_avg:155.56ms
step:129/1480 train_time:18503ms step_avg:155.49ms
step:130/1480 train_time:18649ms step_avg:155.41ms
step:131/1480 train_time:18794ms step_avg:155.32ms
step:132/1480 train_time:18940ms step_avg:155.24ms
step:133/1480 train_time:19086ms step_avg:155.17ms
step:134/1480 train_time:19233ms step_avg:155.10ms
step:135/1480 train_time:19379ms step_avg:155.03ms
step:136/1480 train_time:19526ms step_avg:154.97ms
step:137/1480 train_time:19671ms step_avg:154.89ms
step:138/1480 train_time:19816ms step_avg:154.81ms
step:139/1480 train_time:19962ms step_avg:154.75ms
step:140/1480 train_time:20109ms step_avg:154.68ms
step:141/1480 train_time:20253ms step_avg:154.60ms
step:142/1480 train_time:20399ms step_avg:154.54ms
step:143/1480 train_time:20548ms step_avg:154.49ms
step:144/1480 train_time:20692ms step_avg:154.42ms
step:145/1480 train_time:20838ms step_avg:154.35ms
step:146/1480 train_time:20984ms step_avg:154.29ms
step:147/1480 train_time:21130ms step_avg:154.23ms
step:148/1480 train_time:21275ms step_avg:154.16ms
step:149/1480 train_time:21421ms step_avg:154.11ms
step:150/1480 train_time:21568ms step_avg:154.06ms
step:151/1480 train_time:21713ms step_avg:153.99ms
step:152/1480 train_time:21859ms step_avg:153.93ms
step:153/1480 train_time:22005ms step_avg:153.88ms
step:154/1480 train_time:22150ms step_avg:153.82ms
step:155/1480 train_time:22296ms step_avg:153.76ms
step:156/1480 train_time:22442ms step_avg:153.71ms
step:157/1480 train_time:22589ms step_avg:153.67ms
step:158/1480 train_time:22735ms step_avg:153.61ms
step:159/1480 train_time:22882ms step_avg:153.57ms
step:160/1480 train_time:23028ms step_avg:153.52ms
step:161/1480 train_time:23173ms step_avg:153.46ms
step:162/1480 train_time:23319ms step_avg:153.42ms
step:163/1480 train_time:23466ms step_avg:153.37ms
step:164/1480 train_time:23612ms step_avg:153.32ms
step:165/1480 train_time:23757ms step_avg:153.27ms
step:166/1480 train_time:23904ms step_avg:153.23ms
step:167/1480 train_time:24050ms step_avg:153.19ms
step:168/1480 train_time:24195ms step_avg:153.13ms
step:169/1480 train_time:24342ms step_avg:153.09ms
step:170/1480 train_time:24488ms step_avg:153.05ms
step:171/1480 train_time:24633ms step_avg:153.00ms
step:172/1480 train_time:24779ms step_avg:152.96ms
step:173/1480 train_time:24926ms step_avg:152.92ms
step:174/1480 train_time:25071ms step_avg:152.87ms
step:175/1480 train_time:25218ms step_avg:152.83ms
step:176/1480 train_time:25365ms step_avg:152.80ms
step:177/1480 train_time:25511ms step_avg:152.76ms
step:178/1480 train_time:25655ms step_avg:152.71ms
step:179/1480 train_time:25802ms step_avg:152.67ms
step:180/1480 train_time:25949ms step_avg:152.64ms
step:181/1480 train_time:26093ms step_avg:152.59ms
step:182/1480 train_time:26240ms step_avg:152.56ms
step:183/1480 train_time:26386ms step_avg:152.52ms
step:184/1480 train_time:26531ms step_avg:152.48ms
step:185/1480 train_time:26677ms step_avg:152.44ms
step:186/1480 train_time:26825ms step_avg:152.41ms
step:187/1480 train_time:26971ms step_avg:152.38ms
step:188/1480 train_time:27117ms step_avg:152.34ms
step:189/1480 train_time:27279ms step_avg:152.40ms
step:190/1480 train_time:27410ms step_avg:152.28ms
step:191/1480 train_time:27554ms step_avg:152.23ms
step:192/1480 train_time:27700ms step_avg:152.20ms
step:193/1480 train_time:27847ms step_avg:152.17ms
step:194/1480 train_time:27992ms step_avg:152.13ms
step:195/1480 train_time:28138ms step_avg:152.10ms
step:196/1480 train_time:28285ms step_avg:152.07ms
step:197/1480 train_time:28431ms step_avg:152.04ms
step:198/1480 train_time:28578ms step_avg:152.01ms
step:199/1480 train_time:28725ms step_avg:151.98ms
step:200/1480 train_time:28871ms step_avg:151.95ms
step:201/1480 train_time:29017ms step_avg:151.92ms
step:202/1480 train_time:29162ms step_avg:151.88ms
step:203/1480 train_time:29308ms step_avg:151.85ms
step:204/1480 train_time:29452ms step_avg:151.81ms
step:205/1480 train_time:29598ms step_avg:151.78ms
step:206/1480 train_time:29745ms step_avg:151.76ms
step:207/1480 train_time:29890ms step_avg:151.73ms
step:208/1480 train_time:30036ms step_avg:151.70ms
step:209/1480 train_time:30182ms step_avg:151.67ms
step:210/1480 train_time:30328ms step_avg:151.64ms
step:211/1480 train_time:30472ms step_avg:151.60ms
step:212/1480 train_time:30618ms step_avg:151.57ms
step:213/1480 train_time:30764ms step_avg:151.55ms
step:214/1480 train_time:30910ms step_avg:151.52ms
step:215/1480 train_time:31055ms step_avg:151.49ms
step:216/1480 train_time:31202ms step_avg:151.47ms
step:217/1480 train_time:31349ms step_avg:151.44ms
step:218/1480 train_time:31879ms step_avg:153.26ms
step:219/1480 train_time:31984ms step_avg:153.03ms
step:220/1480 train_time:32130ms step_avg:153.00ms
step:221/1480 train_time:32658ms step_avg:154.78ms
step:222/1480 train_time:32768ms step_avg:154.57ms
step:223/1480 train_time:32915ms step_avg:154.53ms
step:224/1480 train_time:33065ms step_avg:154.51ms
step:225/1480 train_time:33212ms step_avg:154.47ms
step:226/1480 train_time:33359ms step_avg:154.44ms
step:227/1480 train_time:33508ms step_avg:154.41ms
step:228/1480 train_time:33655ms step_avg:154.38ms
step:229/1480 train_time:33805ms step_avg:154.36ms
step:230/1480 train_time:33953ms step_avg:154.33ms
step:231/1480 train_time:34101ms step_avg:154.30ms
step:232/1480 train_time:34250ms step_avg:154.28ms
step:233/1480 train_time:34396ms step_avg:154.24ms
step:234/1480 train_time:34546ms step_avg:154.22ms
step:235/1480 train_time:34693ms step_avg:154.19ms
step:236/1480 train_time:34842ms step_avg:154.17ms
step:237/1480 train_time:34990ms step_avg:154.14ms
step:238/1480 train_time:35139ms step_avg:154.12ms
step:239/1480 train_time:35288ms step_avg:154.10ms
step:240/1480 train_time:35436ms step_avg:154.07ms
step:241/1480 train_time:35585ms step_avg:154.05ms
step:242/1480 train_time:35732ms step_avg:154.02ms
step:243/1480 train_time:35881ms step_avg:153.99ms
step:244/1480 train_time:36029ms step_avg:153.97ms
step:245/1480 train_time:36177ms step_avg:153.95ms
step:246/1480 train_time:36326ms step_avg:153.92ms
step:247/1480 train_time:36473ms step_avg:153.90ms
step:248/1480 train_time:36622ms step_avg:153.88ms
step:249/1480 train_time:36771ms step_avg:153.85ms
step:250/1480 train_time:36919ms step_avg:153.83ms
step:250/1480 val_loss:3.9927 train_time:36986ms step_avg:154.11ms
step:251/1480 train_time:37086ms step_avg:153.88ms
step:252/1480 train_time:37225ms step_avg:153.82ms
step:253/1480 train_time:37373ms step_avg:153.80ms
step:254/1480 train_time:37521ms step_avg:153.78ms
step:255/1480 train_time:37669ms step_avg:153.75ms
step:256/1480 train_time:37817ms step_avg:153.73ms
step:257/1480 train_time:37965ms step_avg:153.71ms
step:258/1480 train_time:38113ms step_avg:153.68ms
step:259/1480 train_time:38263ms step_avg:153.67ms
step:260/1480 train_time:38411ms step_avg:153.64ms
step:261/1480 train_time:38560ms step_avg:153.63ms
step:262/1480 train_time:38710ms step_avg:153.61ms
step:263/1480 train_time:38857ms step_avg:153.59ms
step:264/1480 train_time:39007ms step_avg:153.57ms
step:265/1480 train_time:39155ms step_avg:153.55ms
step:266/1480 train_time:39304ms step_avg:153.53ms
step:267/1480 train_time:39451ms step_avg:153.51ms
step:268/1480 train_time:39600ms step_avg:153.49ms
step:269/1480 train_time:39748ms step_avg:153.47ms
step:270/1480 train_time:39896ms step_avg:153.44ms
step:271/1480 train_time:40045ms step_avg:153.43ms
step:272/1480 train_time:40192ms step_avg:153.40ms
step:273/1480 train_time:40341ms step_avg:153.39ms
step:274/1480 train_time:40490ms step_avg:153.37ms
step:275/1480 train_time:40639ms step_avg:153.35ms
step:276/1480 train_time:40787ms step_avg:153.34ms
step:277/1480 train_time:40935ms step_avg:153.31ms
step:278/1480 train_time:41084ms step_avg:153.30ms
step:279/1480 train_time:41232ms step_avg:153.28ms
step:280/1480 train_time:41382ms step_avg:153.26ms
step:281/1480 train_time:41530ms step_avg:153.25ms
step:282/1480 train_time:41678ms step_avg:153.23ms
step:283/1480 train_time:41829ms step_avg:153.22ms
step:284/1480 train_time:41976ms step_avg:153.20ms
step:285/1480 train_time:42125ms step_avg:153.18ms
step:286/1480 train_time:42273ms step_avg:153.16ms
step:287/1480 train_time:42423ms step_avg:153.15ms
step:288/1480 train_time:42570ms step_avg:153.13ms
step:289/1480 train_time:42719ms step_avg:153.11ms
step:290/1480 train_time:42868ms step_avg:153.10ms
step:291/1480 train_time:43016ms step_avg:153.08ms
step:292/1480 train_time:43165ms step_avg:153.07ms
step:293/1480 train_time:43312ms step_avg:153.05ms
step:294/1480 train_time:43461ms step_avg:153.03ms
step:295/1480 train_time:43610ms step_avg:153.02ms
step:296/1480 train_time:43758ms step_avg:153.00ms
step:297/1480 train_time:43908ms step_avg:152.99ms
step:298/1480 train_time:44055ms step_avg:152.97ms
step:299/1480 train_time:44204ms step_avg:152.96ms
step:300/1480 train_time:44352ms step_avg:152.94ms
step:301/1480 train_time:44503ms step_avg:152.93ms
step:302/1480 train_time:44650ms step_avg:152.91ms
step:303/1480 train_time:44798ms step_avg:152.90ms
step:304/1480 train_time:44947ms step_avg:152.88ms
step:305/1480 train_time:45094ms step_avg:152.86ms
step:306/1480 train_time:45244ms step_avg:152.85ms
step:307/1480 train_time:45391ms step_avg:152.83ms
step:308/1480 train_time:45540ms step_avg:152.82ms
step:309/1480 train_time:45689ms step_avg:152.81ms
step:310/1480 train_time:45838ms step_avg:152.79ms
step:311/1480 train_time:45987ms step_avg:152.78ms
step:312/1480 train_time:46135ms step_avg:152.76ms
step:313/1480 train_time:46284ms step_avg:152.75ms
step:314/1480 train_time:46431ms step_avg:152.73ms
step:315/1480 train_time:46580ms step_avg:152.72ms
step:316/1480 train_time:46729ms step_avg:152.71ms
step:317/1480 train_time:46878ms step_avg:152.70ms
step:318/1480 train_time:47026ms step_avg:152.68ms
step:319/1480 train_time:47174ms step_avg:152.67ms
step:320/1480 train_time:47323ms step_avg:152.66ms
step:321/1480 train_time:47471ms step_avg:152.64ms
step:322/1480 train_time:47620ms step_avg:152.63ms
step:323/1480 train_time:47769ms step_avg:152.62ms
step:324/1480 train_time:47917ms step_avg:152.60ms
step:325/1480 train_time:48066ms step_avg:152.59ms
step:326/1480 train_time:48213ms step_avg:152.57ms
step:327/1480 train_time:48363ms step_avg:152.56ms
step:328/1480 train_time:48510ms step_avg:152.55ms
step:329/1480 train_time:48658ms step_avg:152.53ms
step:330/1480 train_time:48808ms step_avg:152.53ms
step:331/1480 train_time:48960ms step_avg:152.52ms
step:332/1480 train_time:49110ms step_avg:152.52ms
step:333/1480 train_time:49262ms step_avg:152.51ms
step:334/1480 train_time:49412ms step_avg:152.51ms
step:335/1480 train_time:49564ms step_avg:152.50ms
step:336/1480 train_time:49714ms step_avg:152.50ms
step:337/1480 train_time:49866ms step_avg:152.49ms
step:338/1480 train_time:50015ms step_avg:152.49ms
step:339/1480 train_time:50167ms step_avg:152.48ms
step:340/1480 train_time:50317ms step_avg:152.47ms
step:341/1480 train_time:50468ms step_avg:152.47ms
step:342/1480 train_time:50618ms step_avg:152.46ms
step:343/1480 train_time:50769ms step_avg:152.46ms
step:344/1480 train_time:50919ms step_avg:152.45ms
step:345/1480 train_time:51070ms step_avg:152.45ms
step:346/1480 train_time:51220ms step_avg:152.44ms
step:347/1480 train_time:51371ms step_avg:152.44ms
step:348/1480 train_time:51522ms step_avg:152.43ms
step:349/1480 train_time:51673ms step_avg:152.43ms
step:350/1480 train_time:51824ms step_avg:152.42ms
step:351/1480 train_time:51975ms step_avg:152.42ms
step:352/1480 train_time:52126ms step_avg:152.42ms
step:353/1480 train_time:52276ms step_avg:152.41ms
step:354/1480 train_time:52428ms step_avg:152.41ms
step:355/1480 train_time:52578ms step_avg:152.40ms
step:356/1480 train_time:52729ms step_avg:152.40ms
step:357/1480 train_time:52880ms step_avg:152.39ms
step:358/1480 train_time:53031ms step_avg:152.39ms
step:359/1480 train_time:53184ms step_avg:152.39ms
step:360/1480 train_time:53334ms step_avg:152.38ms
step:361/1480 train_time:53486ms step_avg:152.38ms
step:362/1480 train_time:53636ms step_avg:152.38ms
step:363/1480 train_time:53787ms step_avg:152.37ms
step:364/1480 train_time:53937ms step_avg:152.36ms
step:365/1480 train_time:54089ms step_avg:152.36ms
step:366/1480 train_time:54239ms step_avg:152.36ms
step:367/1480 train_time:54390ms step_avg:152.35ms
step:368/1480 train_time:54541ms step_avg:152.35ms
step:369/1480 train_time:54691ms step_avg:152.34ms
step:370/1480 train_time:54843ms step_avg:152.34ms
step:371/1480 train_time:54993ms step_avg:152.34ms
step:372/1480 train_time:55146ms step_avg:152.34ms
step:373/1480 train_time:55296ms step_avg:152.33ms
step:374/1480 train_time:55448ms step_avg:152.33ms
step:375/1480 train_time:55597ms step_avg:152.32ms
step:375/1480 val_loss:3.8055 train_time:55666ms step_avg:152.51ms
step:376/1480 train_time:55762ms step_avg:152.36ms
step:377/1480 train_time:55909ms step_avg:152.34ms
step:378/1480 train_time:56060ms step_avg:152.34ms
step:379/1480 train_time:56222ms step_avg:152.36ms
step:380/1480 train_time:56360ms step_avg:152.32ms
step:381/1480 train_time:56510ms step_avg:152.32ms
step:382/1480 train_time:56662ms step_avg:152.32ms
step:383/1480 train_time:56812ms step_avg:152.31ms
step:384/1480 train_time:56965ms step_avg:152.31ms
step:385/1480 train_time:57115ms step_avg:152.31ms
step:386/1480 train_time:57267ms step_avg:152.31ms
step:387/1480 train_time:57417ms step_avg:152.30ms
step:388/1480 train_time:57568ms step_avg:152.30ms
step:389/1480 train_time:57718ms step_avg:152.29ms
step:390/1480 train_time:57869ms step_avg:152.29ms
step:391/1480 train_time:58021ms step_avg:152.29ms
step:392/1480 train_time:58171ms step_avg:152.28ms
step:393/1480 train_time:58323ms step_avg:152.28ms
step:394/1480 train_time:58473ms step_avg:152.27ms
step:395/1480 train_time:58624ms step_avg:152.27ms
step:396/1480 train_time:58776ms step_avg:152.27ms
step:397/1480 train_time:58927ms step_avg:152.27ms
step:398/1480 train_time:59078ms step_avg:152.26ms
step:399/1480 train_time:59228ms step_avg:152.26ms
step:400/1480 train_time:59380ms step_avg:152.26ms
step:401/1480 train_time:59531ms step_avg:152.25ms
step:402/1480 train_time:59683ms step_avg:152.25ms
step:403/1480 train_time:59833ms step_avg:152.25ms
step:404/1480 train_time:59985ms step_avg:152.25ms
step:405/1480 train_time:60135ms step_avg:152.24ms
step:406/1480 train_time:60286ms step_avg:152.24ms
step:407/1480 train_time:60436ms step_avg:152.23ms
step:408/1480 train_time:60587ms step_avg:152.23ms
step:409/1480 train_time:60738ms step_avg:152.23ms
step:410/1480 train_time:60889ms step_avg:152.22ms
step:411/1480 train_time:61040ms step_avg:152.22ms
step:412/1480 train_time:61190ms step_avg:152.21ms
step:413/1480 train_time:61341ms step_avg:152.21ms
step:414/1480 train_time:61491ms step_avg:152.21ms
step:415/1480 train_time:61643ms step_avg:152.20ms
step:416/1480 train_time:61793ms step_avg:152.20ms
step:417/1480 train_time:61945ms step_avg:152.20ms
step:418/1480 train_time:62096ms step_avg:152.20ms
step:419/1480 train_time:62247ms step_avg:152.19ms
step:420/1480 train_time:62398ms step_avg:152.19ms
step:421/1480 train_time:62549ms step_avg:152.19ms
step:422/1480 train_time:62702ms step_avg:152.19ms
step:423/1480 train_time:62852ms step_avg:152.18ms
step:424/1480 train_time:63004ms step_avg:152.18ms
step:425/1480 train_time:63154ms step_avg:152.18ms
step:426/1480 train_time:63305ms step_avg:152.18ms
step:427/1480 train_time:63455ms step_avg:152.17ms
step:428/1480 train_time:63607ms step_avg:152.17ms
step:429/1480 train_time:63757ms step_avg:152.16ms
step:430/1480 train_time:63908ms step_avg:152.16ms
step:431/1480 train_time:64060ms step_avg:152.16ms
step:432/1480 train_time:64211ms step_avg:152.16ms
step:433/1480 train_time:64363ms step_avg:152.16ms
step:434/1480 train_time:64513ms step_avg:152.15ms
step:435/1480 train_time:64665ms step_avg:152.15ms
step:436/1480 train_time:64815ms step_avg:152.15ms
step:437/1480 train_time:64966ms step_avg:152.15ms
step:438/1480 train_time:65116ms step_avg:152.14ms
step:439/1480 train_time:65268ms step_avg:152.14ms
step:440/1480 train_time:65418ms step_avg:152.14ms
step:441/1480 train_time:65570ms step_avg:152.14ms
step:442/1480 train_time:65724ms step_avg:152.14ms
step:443/1480 train_time:65877ms step_avg:152.14ms
step:444/1480 train_time:66029ms step_avg:152.14ms
step:445/1480 train_time:66183ms step_avg:152.15ms
step:446/1480 train_time:66336ms step_avg:152.15ms
step:447/1480 train_time:66488ms step_avg:152.15ms
step:448/1480 train_time:66642ms step_avg:152.15ms
step:449/1480 train_time:66794ms step_avg:152.15ms
step:450/1480 train_time:66946ms step_avg:152.15ms
step:451/1480 train_time:67100ms step_avg:152.15ms
step:452/1480 train_time:67253ms step_avg:152.16ms
step:453/1480 train_time:67407ms step_avg:152.16ms
step:454/1480 train_time:67560ms step_avg:152.16ms
step:455/1480 train_time:67712ms step_avg:152.16ms
step:456/1480 train_time:67866ms step_avg:152.17ms
step:457/1480 train_time:68018ms step_avg:152.17ms
step:458/1480 train_time:68171ms step_avg:152.17ms
step:459/1480 train_time:68324ms step_avg:152.17ms
step:460/1480 train_time:68478ms step_avg:152.17ms
step:461/1480 train_time:68631ms step_avg:152.17ms
step:462/1480 train_time:68785ms step_avg:152.18ms
step:463/1480 train_time:68937ms step_avg:152.18ms
step:464/1480 train_time:69091ms step_avg:152.18ms
step:465/1480 train_time:69244ms step_avg:152.18ms
step:466/1480 train_time:69396ms step_avg:152.19ms
step:467/1480 train_time:69549ms step_avg:152.19ms
step:468/1480 train_time:69702ms step_avg:152.19ms
step:469/1480 train_time:69855ms step_avg:152.19ms
step:470/1480 train_time:70008ms step_avg:152.19ms
step:471/1480 train_time:70161ms step_avg:152.19ms
step:472/1480 train_time:70314ms step_avg:152.19ms
step:473/1480 train_time:70468ms step_avg:152.20ms
step:474/1480 train_time:70620ms step_avg:152.20ms
step:475/1480 train_time:70773ms step_avg:152.20ms
step:476/1480 train_time:70927ms step_avg:152.20ms
step:477/1480 train_time:71079ms step_avg:152.20ms
step:478/1480 train_time:71232ms step_avg:152.21ms
step:479/1480 train_time:71386ms step_avg:152.21ms
step:480/1480 train_time:71538ms step_avg:152.21ms
step:481/1480 train_time:71691ms step_avg:152.21ms
step:482/1480 train_time:71845ms step_avg:152.21ms
step:483/1480 train_time:71997ms step_avg:152.21ms
step:484/1480 train_time:72150ms step_avg:152.22ms
step:485/1480 train_time:72304ms step_avg:152.22ms
step:486/1480 train_time:72457ms step_avg:152.22ms
step:487/1480 train_time:72610ms step_avg:152.22ms
step:488/1480 train_time:72762ms step_avg:152.22ms
step:489/1480 train_time:72914ms step_avg:152.22ms
step:490/1480 train_time:73068ms step_avg:152.22ms
step:491/1480 train_time:73220ms step_avg:152.22ms
step:492/1480 train_time:73373ms step_avg:152.23ms
step:493/1480 train_time:73527ms step_avg:152.23ms
step:494/1480 train_time:73679ms step_avg:152.23ms
step:495/1480 train_time:73833ms step_avg:152.23ms
step:496/1480 train_time:73987ms step_avg:152.24ms
step:497/1480 train_time:74139ms step_avg:152.24ms
step:498/1480 train_time:74291ms step_avg:152.24ms
step:499/1480 train_time:74444ms step_avg:152.24ms
step:500/1480 train_time:74598ms step_avg:152.24ms
step:500/1480 val_loss:3.6852 train_time:74668ms step_avg:152.38ms
step:501/1480 train_time:74764ms step_avg:152.27ms
step:502/1480 train_time:74913ms step_avg:152.26ms
step:503/1480 train_time:75065ms step_avg:152.26ms
step:504/1480 train_time:75218ms step_avg:152.26ms
step:505/1480 train_time:75371ms step_avg:152.26ms
step:506/1480 train_time:75522ms step_avg:152.26ms
step:507/1480 train_time:75675ms step_avg:152.26ms
step:508/1480 train_time:75829ms step_avg:152.27ms
step:509/1480 train_time:75983ms step_avg:152.27ms
step:510/1480 train_time:76136ms step_avg:152.27ms
step:511/1480 train_time:76290ms step_avg:152.28ms
step:512/1480 train_time:76443ms step_avg:152.28ms
step:513/1480 train_time:76595ms step_avg:152.28ms
step:514/1480 train_time:76749ms step_avg:152.28ms
step:515/1480 train_time:76902ms step_avg:152.28ms
step:516/1480 train_time:77056ms step_avg:152.28ms
step:517/1480 train_time:77210ms step_avg:152.29ms
step:518/1480 train_time:77362ms step_avg:152.29ms
step:519/1480 train_time:77515ms step_avg:152.29ms
step:520/1480 train_time:77669ms step_avg:152.29ms
step:521/1480 train_time:77821ms step_avg:152.29ms
step:522/1480 train_time:77974ms step_avg:152.29ms
step:523/1480 train_time:78127ms step_avg:152.29ms
step:524/1480 train_time:78281ms step_avg:152.30ms
step:525/1480 train_time:78434ms step_avg:152.30ms
step:526/1480 train_time:78587ms step_avg:152.30ms
step:527/1480 train_time:78739ms step_avg:152.30ms
step:528/1480 train_time:78892ms step_avg:152.30ms
step:529/1480 train_time:79045ms step_avg:152.30ms
step:530/1480 train_time:79198ms step_avg:152.30ms
step:531/1480 train_time:79351ms step_avg:152.31ms
step:532/1480 train_time:79505ms step_avg:152.31ms
step:533/1480 train_time:79659ms step_avg:152.31ms
step:534/1480 train_time:79812ms step_avg:152.31ms
step:535/1480 train_time:79964ms step_avg:152.31ms
step:536/1480 train_time:80116ms step_avg:152.31ms
step:537/1480 train_time:80270ms step_avg:152.31ms
step:538/1480 train_time:80423ms step_avg:152.32ms
step:539/1480 train_time:80577ms step_avg:152.32ms
step:540/1480 train_time:80732ms step_avg:152.32ms
step:541/1480 train_time:80884ms step_avg:152.32ms
step:542/1480 train_time:81037ms step_avg:152.33ms
step:543/1480 train_time:81191ms step_avg:152.33ms
step:544/1480 train_time:81343ms step_avg:152.33ms
step:545/1480 train_time:81496ms step_avg:152.33ms
step:546/1480 train_time:81650ms step_avg:152.33ms
step:547/1480 train_time:81804ms step_avg:152.34ms
step:548/1480 train_time:81957ms step_avg:152.34ms
step:549/1480 train_time:82111ms step_avg:152.34ms
step:550/1480 train_time:82264ms step_avg:152.34ms
step:551/1480 train_time:82418ms step_avg:152.34ms
step:552/1480 train_time:82574ms step_avg:152.35ms
step:553/1480 train_time:82728ms step_avg:152.35ms
step:554/1480 train_time:82884ms step_avg:152.36ms
step:555/1480 train_time:83038ms step_avg:152.36ms
step:556/1480 train_time:83193ms step_avg:152.37ms
step:557/1480 train_time:83347ms step_avg:152.37ms
step:558/1480 train_time:83502ms step_avg:152.38ms
step:559/1480 train_time:83656ms step_avg:152.38ms
step:560/1480 train_time:83812ms step_avg:152.38ms
step:561/1480 train_time:83966ms step_avg:152.39ms
step:562/1480 train_time:84120ms step_avg:152.39ms
step:563/1480 train_time:84274ms step_avg:152.39ms
step:564/1480 train_time:84429ms step_avg:152.40ms
step:565/1480 train_time:84584ms step_avg:152.40ms
step:566/1480 train_time:84740ms step_avg:152.41ms
step:567/1480 train_time:84895ms step_avg:152.41ms
step:568/1480 train_time:85049ms step_avg:152.42ms
step:569/1480 train_time:85218ms step_avg:152.45ms
step:570/1480 train_time:85360ms step_avg:152.43ms
step:571/1480 train_time:85515ms step_avg:152.43ms
step:572/1480 train_time:85669ms step_avg:152.44ms
step:573/1480 train_time:85824ms step_avg:152.44ms
step:574/1480 train_time:85979ms step_avg:152.44ms
step:575/1480 train_time:86134ms step_avg:152.45ms
step:576/1480 train_time:86289ms step_avg:152.45ms
step:577/1480 train_time:86444ms step_avg:152.46ms
step:578/1480 train_time:86599ms step_avg:152.46ms
step:579/1480 train_time:86753ms step_avg:152.47ms
step:580/1480 train_time:86908ms step_avg:152.47ms
step:581/1480 train_time:87062ms step_avg:152.47ms
step:582/1480 train_time:87217ms step_avg:152.48ms
step:583/1480 train_time:87373ms step_avg:152.48ms
step:584/1480 train_time:87527ms step_avg:152.49ms
step:585/1480 train_time:87682ms step_avg:152.49ms
step:586/1480 train_time:87837ms step_avg:152.49ms
step:587/1480 train_time:87992ms step_avg:152.50ms
step:588/1480 train_time:88146ms step_avg:152.50ms
step:589/1480 train_time:88300ms step_avg:152.50ms
step:590/1480 train_time:88454ms step_avg:152.51ms
step:591/1480 train_time:88610ms step_avg:152.51ms
step:592/1480 train_time:88765ms step_avg:152.52ms
step:593/1480 train_time:88921ms step_avg:152.52ms
step:594/1480 train_time:89076ms step_avg:152.53ms
step:595/1480 train_time:89233ms step_avg:152.53ms
step:596/1480 train_time:89389ms step_avg:152.54ms
step:597/1480 train_time:89544ms step_avg:152.54ms
step:598/1480 train_time:89698ms step_avg:152.55ms
step:599/1480 train_time:89852ms step_avg:152.55ms
step:600/1480 train_time:90007ms step_avg:152.56ms
step:601/1480 train_time:90163ms step_avg:152.56ms
step:602/1480 train_time:90317ms step_avg:152.56ms
step:603/1480 train_time:90474ms step_avg:152.57ms
step:604/1480 train_time:90628ms step_avg:152.57ms
step:605/1480 train_time:90783ms step_avg:152.58ms
step:606/1480 train_time:90939ms step_avg:152.58ms
step:607/1480 train_time:91095ms step_avg:152.59ms
step:608/1480 train_time:91250ms step_avg:152.59ms
step:609/1480 train_time:91405ms step_avg:152.60ms
step:610/1480 train_time:91559ms step_avg:152.60ms
step:611/1480 train_time:91714ms step_avg:152.60ms
step:612/1480 train_time:91870ms step_avg:152.61ms
step:613/1480 train_time:92024ms step_avg:152.61ms
step:614/1480 train_time:92180ms step_avg:152.62ms
step:615/1480 train_time:92335ms step_avg:152.62ms
step:616/1480 train_time:92489ms step_avg:152.62ms
step:617/1480 train_time:92643ms step_avg:152.62ms
step:618/1480 train_time:92797ms step_avg:152.63ms
step:619/1480 train_time:92953ms step_avg:152.63ms
step:620/1480 train_time:93108ms step_avg:152.64ms
step:621/1480 train_time:93262ms step_avg:152.64ms
step:622/1480 train_time:93417ms step_avg:152.64ms
step:623/1480 train_time:93573ms step_avg:152.65ms
step:624/1480 train_time:93728ms step_avg:152.65ms
step:625/1480 train_time:93882ms step_avg:152.65ms
step:625/1480 val_loss:3.6050 train_time:93954ms step_avg:152.77ms
step:626/1480 train_time:94051ms step_avg:152.68ms
step:627/1480 train_time:94200ms step_avg:152.67ms
step:628/1480 train_time:94355ms step_avg:152.68ms
step:629/1480 train_time:94510ms step_avg:152.68ms
step:630/1480 train_time:94664ms step_avg:152.68ms
step:631/1480 train_time:94818ms step_avg:152.69ms
step:632/1480 train_time:94972ms step_avg:152.69ms
step:633/1480 train_time:95126ms step_avg:152.69ms
step:634/1480 train_time:95282ms step_avg:152.70ms
step:635/1480 train_time:95438ms step_avg:152.70ms
step:636/1480 train_time:95592ms step_avg:152.70ms
step:637/1480 train_time:95747ms step_avg:152.71ms
step:638/1480 train_time:95902ms step_avg:152.71ms
step:639/1480 train_time:96058ms step_avg:152.72ms
step:640/1480 train_time:96212ms step_avg:152.72ms
step:641/1480 train_time:96369ms step_avg:152.72ms
step:642/1480 train_time:96523ms step_avg:152.73ms
step:643/1480 train_time:96677ms step_avg:152.73ms
step:644/1480 train_time:96831ms step_avg:152.73ms
step:645/1480 train_time:96988ms step_avg:152.74ms
step:646/1480 train_time:97142ms step_avg:152.74ms
step:647/1480 train_time:97297ms step_avg:152.74ms
step:648/1480 train_time:97452ms step_avg:152.75ms
step:649/1480 train_time:97608ms step_avg:152.75ms
step:650/1480 train_time:97762ms step_avg:152.75ms
step:651/1480 train_time:97918ms step_avg:152.76ms
step:652/1480 train_time:98072ms step_avg:152.76ms
step:653/1480 train_time:98227ms step_avg:152.76ms
step:654/1480 train_time:98382ms step_avg:152.77ms
step:655/1480 train_time:98538ms step_avg:152.77ms
step:656/1480 train_time:98692ms step_avg:152.77ms
step:657/1480 train_time:98847ms step_avg:152.78ms
step:658/1480 train_time:99002ms step_avg:152.78ms
step:659/1480 train_time:99157ms step_avg:152.78ms
step:660/1480 train_time:99313ms step_avg:152.79ms
step:661/1480 train_time:99471ms step_avg:152.80ms
step:662/1480 train_time:99628ms step_avg:152.80ms
step:663/1480 train_time:99784ms step_avg:152.81ms
step:664/1480 train_time:99939ms step_avg:152.81ms
step:665/1480 train_time:100095ms step_avg:152.82ms
step:666/1480 train_time:100251ms step_avg:152.82ms
step:667/1480 train_time:100408ms step_avg:152.83ms
step:668/1480 train_time:100564ms step_avg:152.83ms
step:669/1480 train_time:100721ms step_avg:152.84ms
step:670/1480 train_time:100877ms step_avg:152.84ms
step:671/1480 train_time:101033ms step_avg:152.85ms
step:672/1480 train_time:101190ms step_avg:152.85ms
step:673/1480 train_time:101345ms step_avg:152.86ms
step:674/1480 train_time:101501ms step_avg:152.86ms
step:675/1480 train_time:101659ms step_avg:152.87ms
step:676/1480 train_time:101815ms step_avg:152.88ms
step:677/1480 train_time:101971ms step_avg:152.88ms
step:678/1480 train_time:102128ms step_avg:152.89ms
step:679/1480 train_time:102284ms step_avg:152.89ms
step:680/1480 train_time:102442ms step_avg:152.90ms
step:681/1480 train_time:102598ms step_avg:152.90ms
step:682/1480 train_time:102754ms step_avg:152.91ms
step:683/1480 train_time:102910ms step_avg:152.91ms
step:684/1480 train_time:103067ms step_avg:152.92ms
step:685/1480 train_time:103222ms step_avg:152.92ms
step:686/1480 train_time:103380ms step_avg:152.93ms
step:687/1480 train_time:103535ms step_avg:152.93ms
step:688/1480 train_time:103693ms step_avg:152.94ms
step:689/1480 train_time:103850ms step_avg:152.95ms
step:690/1480 train_time:104008ms step_avg:152.95ms
step:691/1480 train_time:104164ms step_avg:152.96ms
step:692/1480 train_time:104320ms step_avg:152.96ms
step:693/1480 train_time:104477ms step_avg:152.97ms
step:694/1480 train_time:104634ms step_avg:152.97ms
step:695/1480 train_time:104790ms step_avg:152.98ms
step:696/1480 train_time:104945ms step_avg:152.98ms
step:697/1480 train_time:105102ms step_avg:152.99ms
step:698/1480 train_time:105258ms step_avg:152.99ms
step:699/1480 train_time:105415ms step_avg:153.00ms
step:700/1480 train_time:105570ms step_avg:153.00ms
step:701/1480 train_time:105726ms step_avg:153.00ms
step:702/1480 train_time:105882ms step_avg:153.01ms
step:703/1480 train_time:106038ms step_avg:153.01ms
step:704/1480 train_time:106194ms step_avg:153.02ms
step:705/1480 train_time:106351ms step_avg:153.02ms
step:706/1480 train_time:106511ms step_avg:153.03ms
step:707/1480 train_time:106669ms step_avg:153.04ms
step:708/1480 train_time:106823ms step_avg:153.04ms
step:709/1480 train_time:106978ms step_avg:153.04ms
step:710/1480 train_time:107133ms step_avg:153.05ms
step:711/1480 train_time:107291ms step_avg:153.05ms
step:712/1480 train_time:107448ms step_avg:153.06ms
step:713/1480 train_time:107606ms step_avg:153.07ms
step:714/1480 train_time:107763ms step_avg:153.07ms
step:715/1480 train_time:107918ms step_avg:153.07ms
step:716/1480 train_time:108073ms step_avg:153.08ms
step:717/1480 train_time:108230ms step_avg:153.08ms
step:718/1480 train_time:108387ms step_avg:153.09ms
step:719/1480 train_time:108541ms step_avg:153.09ms
step:720/1480 train_time:108700ms step_avg:153.10ms
step:721/1480 train_time:108858ms step_avg:153.11ms
step:722/1480 train_time:109014ms step_avg:153.11ms
step:723/1480 train_time:109170ms step_avg:153.11ms
step:724/1480 train_time:109325ms step_avg:153.12ms
step:725/1480 train_time:109482ms step_avg:153.12ms
step:726/1480 train_time:109638ms step_avg:153.13ms
step:727/1480 train_time:109796ms step_avg:153.13ms
step:728/1480 train_time:109952ms step_avg:153.14ms
step:729/1480 train_time:110109ms step_avg:153.14ms
step:730/1480 train_time:110265ms step_avg:153.15ms
step:731/1480 train_time:110421ms step_avg:153.15ms
step:732/1480 train_time:110578ms step_avg:153.16ms
step:733/1480 train_time:110734ms step_avg:153.16ms
step:734/1480 train_time:110891ms step_avg:153.16ms
step:735/1480 train_time:111049ms step_avg:153.17ms
step:736/1480 train_time:111206ms step_avg:153.18ms
step:737/1480 train_time:111361ms step_avg:153.18ms
step:738/1480 train_time:111515ms step_avg:153.18ms
step:739/1480 train_time:111672ms step_avg:153.18ms
step:740/1480 train_time:111831ms step_avg:153.19ms
step:741/1480 train_time:111989ms step_avg:153.20ms
step:742/1480 train_time:112144ms step_avg:153.20ms
step:743/1480 train_time:112300ms step_avg:153.21ms
step:744/1480 train_time:112456ms step_avg:153.21ms
step:745/1480 train_time:112613ms step_avg:153.22ms
step:746/1480 train_time:112770ms step_avg:153.22ms
step:747/1480 train_time:112927ms step_avg:153.22ms
step:748/1480 train_time:113087ms step_avg:153.23ms
step:749/1480 train_time:113243ms step_avg:153.24ms
step:750/1480 train_time:113399ms step_avg:153.24ms
step:750/1480 val_loss:3.5492 train_time:113470ms step_avg:153.34ms
step:751/1480 train_time:113566ms step_avg:153.26ms
step:752/1480 train_time:113717ms step_avg:153.26ms
step:753/1480 train_time:113873ms step_avg:153.26ms
step:754/1480 train_time:114029ms step_avg:153.26ms
step:755/1480 train_time:114185ms step_avg:153.27ms
step:756/1480 train_time:114340ms step_avg:153.27ms
step:757/1480 train_time:114499ms step_avg:153.28ms
step:758/1480 train_time:114656ms step_avg:153.28ms
step:759/1480 train_time:114824ms step_avg:153.30ms
step:760/1480 train_time:114972ms step_avg:153.30ms
step:761/1480 train_time:115128ms step_avg:153.30ms
step:762/1480 train_time:115285ms step_avg:153.30ms
step:763/1480 train_time:115442ms step_avg:153.31ms
step:764/1480 train_time:115599ms step_avg:153.31ms
step:765/1480 train_time:115756ms step_avg:153.32ms
step:766/1480 train_time:115914ms step_avg:153.33ms
step:767/1480 train_time:116071ms step_avg:153.33ms
step:768/1480 train_time:116227ms step_avg:153.33ms
step:769/1480 train_time:116385ms step_avg:153.34ms
step:770/1480 train_time:116543ms step_avg:153.35ms
step:771/1480 train_time:116700ms step_avg:153.35ms
step:772/1480 train_time:116858ms step_avg:153.36ms
step:773/1480 train_time:117016ms step_avg:153.36ms
step:774/1480 train_time:117174ms step_avg:153.37ms
step:775/1480 train_time:117332ms step_avg:153.37ms
step:776/1480 train_time:117492ms step_avg:153.38ms
step:777/1480 train_time:117652ms step_avg:153.39ms
step:778/1480 train_time:117810ms step_avg:153.40ms
step:779/1480 train_time:117968ms step_avg:153.40ms
step:780/1480 train_time:118127ms step_avg:153.41ms
step:781/1480 train_time:118285ms step_avg:153.42ms
step:782/1480 train_time:118444ms step_avg:153.43ms
step:783/1480 train_time:118601ms step_avg:153.43ms
step:784/1480 train_time:118758ms step_avg:153.43ms
step:785/1480 train_time:118915ms step_avg:153.44ms
step:786/1480 train_time:119073ms step_avg:153.44ms
step:787/1480 train_time:119231ms step_avg:153.45ms
step:788/1480 train_time:119392ms step_avg:153.46ms
step:789/1480 train_time:119550ms step_avg:153.47ms
step:790/1480 train_time:119706ms step_avg:153.47ms
step:791/1480 train_time:119869ms step_avg:153.48ms
step:792/1480 train_time:120025ms step_avg:153.48ms
step:793/1480 train_time:120182ms step_avg:153.49ms
step:794/1480 train_time:120341ms step_avg:153.50ms
step:795/1480 train_time:120502ms step_avg:153.51ms
step:796/1480 train_time:120663ms step_avg:153.52ms
step:797/1480 train_time:120822ms step_avg:153.52ms
step:798/1480 train_time:120981ms step_avg:153.53ms
step:799/1480 train_time:121142ms step_avg:153.54ms
step:800/1480 train_time:121300ms step_avg:153.54ms
step:801/1480 train_time:121458ms step_avg:153.55ms
step:802/1480 train_time:121620ms step_avg:153.56ms
step:803/1480 train_time:121777ms step_avg:153.56ms
step:804/1480 train_time:121933ms step_avg:153.57ms
step:805/1480 train_time:122094ms step_avg:153.58ms
step:806/1480 train_time:122251ms step_avg:153.58ms
step:807/1480 train_time:122408ms step_avg:153.59ms
step:808/1480 train_time:122567ms step_avg:153.59ms
step:809/1480 train_time:122725ms step_avg:153.60ms
step:810/1480 train_time:122882ms step_avg:153.60ms
step:811/1480 train_time:123039ms step_avg:153.61ms
step:812/1480 train_time:123195ms step_avg:153.61ms
step:813/1480 train_time:123352ms step_avg:153.61ms
step:814/1480 train_time:123510ms step_avg:153.62ms
step:815/1480 train_time:123666ms step_avg:153.62ms
step:816/1480 train_time:123824ms step_avg:153.63ms
step:817/1480 train_time:123982ms step_avg:153.63ms
step:818/1480 train_time:124140ms step_avg:153.64ms
step:819/1480 train_time:124297ms step_avg:153.64ms
step:820/1480 train_time:124456ms step_avg:153.65ms
step:821/1480 train_time:124613ms step_avg:153.65ms
step:822/1480 train_time:124771ms step_avg:153.66ms
step:823/1480 train_time:124929ms step_avg:153.66ms
step:824/1480 train_time:125087ms step_avg:153.67ms
step:825/1480 train_time:125247ms step_avg:153.68ms
step:826/1480 train_time:125408ms step_avg:153.69ms
step:827/1480 train_time:125568ms step_avg:153.69ms
step:828/1480 train_time:125728ms step_avg:153.70ms
step:829/1480 train_time:125887ms step_avg:153.71ms
step:830/1480 train_time:126048ms step_avg:153.72ms
step:831/1480 train_time:126205ms step_avg:153.72ms
step:832/1480 train_time:126363ms step_avg:153.73ms
step:833/1480 train_time:126520ms step_avg:153.73ms
step:834/1480 train_time:126682ms step_avg:153.74ms
step:835/1480 train_time:126838ms step_avg:153.74ms
step:836/1480 train_time:126998ms step_avg:153.75ms
step:837/1480 train_time:127154ms step_avg:153.75ms
step:838/1480 train_time:127313ms step_avg:153.76ms
step:839/1480 train_time:127471ms step_avg:153.76ms
step:840/1480 train_time:127628ms step_avg:153.77ms
step:841/1480 train_time:127784ms step_avg:153.77ms
step:842/1480 train_time:127944ms step_avg:153.78ms
step:843/1480 train_time:128101ms step_avg:153.78ms
step:844/1480 train_time:128258ms step_avg:153.79ms
step:845/1480 train_time:128415ms step_avg:153.79ms
step:846/1480 train_time:128574ms step_avg:153.80ms
step:847/1480 train_time:128733ms step_avg:153.80ms
step:848/1480 train_time:128891ms step_avg:153.81ms
step:849/1480 train_time:129050ms step_avg:153.81ms
step:850/1480 train_time:129208ms step_avg:153.82ms
step:851/1480 train_time:129368ms step_avg:153.83ms
step:852/1480 train_time:129526ms step_avg:153.83ms
step:853/1480 train_time:129685ms step_avg:153.84ms
step:854/1480 train_time:129842ms step_avg:153.84ms
step:855/1480 train_time:129999ms step_avg:153.85ms
step:856/1480 train_time:130158ms step_avg:153.85ms
step:857/1480 train_time:130315ms step_avg:153.86ms
step:858/1480 train_time:130476ms step_avg:153.86ms
step:859/1480 train_time:130633ms step_avg:153.87ms
step:860/1480 train_time:130792ms step_avg:153.87ms
step:861/1480 train_time:130950ms step_avg:153.88ms
step:862/1480 train_time:131112ms step_avg:153.89ms
step:863/1480 train_time:131270ms step_avg:153.89ms
step:864/1480 train_time:131427ms step_avg:153.90ms
step:865/1480 train_time:131585ms step_avg:153.90ms
step:866/1480 train_time:131744ms step_avg:153.91ms
step:867/1480 train_time:131902ms step_avg:153.91ms
step:868/1480 train_time:132058ms step_avg:153.91ms
step:869/1480 train_time:132215ms step_avg:153.92ms
step:870/1480 train_time:132375ms step_avg:153.92ms
step:871/1480 train_time:132532ms step_avg:153.93ms
step:872/1480 train_time:132690ms step_avg:153.93ms
step:873/1480 train_time:132847ms step_avg:153.94ms
step:874/1480 train_time:133007ms step_avg:153.94ms
step:875/1480 train_time:133167ms step_avg:153.95ms
step:875/1480 val_loss:3.5032 train_time:133240ms step_avg:154.03ms
step:876/1480 train_time:133332ms step_avg:153.96ms
step:877/1480 train_time:133487ms step_avg:153.96ms
step:878/1480 train_time:133646ms step_avg:153.97ms
step:879/1480 train_time:133805ms step_avg:153.98ms
step:880/1480 train_time:133963ms step_avg:153.98ms
step:881/1480 train_time:134121ms step_avg:153.98ms
step:882/1480 train_time:134281ms step_avg:153.99ms
step:883/1480 train_time:134442ms step_avg:154.00ms
step:884/1480 train_time:134603ms step_avg:154.01ms
step:885/1480 train_time:134763ms step_avg:154.01ms
step:886/1480 train_time:134923ms step_avg:154.02ms
step:887/1480 train_time:135081ms step_avg:154.03ms
step:888/1480 train_time:135245ms step_avg:154.04ms
step:889/1480 train_time:135409ms step_avg:154.05ms
step:890/1480 train_time:135567ms step_avg:154.05ms
step:891/1480 train_time:135727ms step_avg:154.06ms
step:892/1480 train_time:135887ms step_avg:154.07ms
step:893/1480 train_time:136045ms step_avg:154.07ms
step:894/1480 train_time:136205ms step_avg:154.08ms
step:895/1480 train_time:136365ms step_avg:154.08ms
step:896/1480 train_time:136524ms step_avg:154.09ms
step:897/1480 train_time:136685ms step_avg:154.10ms
step:898/1480 train_time:136846ms step_avg:154.11ms
step:899/1480 train_time:137006ms step_avg:154.11ms
step:900/1480 train_time:137165ms step_avg:154.12ms
step:901/1480 train_time:137324ms step_avg:154.12ms
step:902/1480 train_time:137481ms step_avg:154.13ms
step:903/1480 train_time:137645ms step_avg:154.14ms
step:904/1480 train_time:137805ms step_avg:154.14ms
step:905/1480 train_time:137963ms step_avg:154.15ms
step:906/1480 train_time:138121ms step_avg:154.15ms
step:907/1480 train_time:138285ms step_avg:154.16ms
step:908/1480 train_time:138443ms step_avg:154.17ms
step:909/1480 train_time:138602ms step_avg:154.17ms
step:910/1480 train_time:138766ms step_avg:154.18ms
step:911/1480 train_time:138927ms step_avg:154.19ms
step:912/1480 train_time:139088ms step_avg:154.20ms
step:913/1480 train_time:139249ms step_avg:154.21ms
step:914/1480 train_time:139410ms step_avg:154.21ms
step:915/1480 train_time:139572ms step_avg:154.22ms
step:916/1480 train_time:139731ms step_avg:154.23ms
step:917/1480 train_time:139889ms step_avg:154.23ms
step:918/1480 train_time:140051ms step_avg:154.24ms
step:919/1480 train_time:140214ms step_avg:154.25ms
step:920/1480 train_time:140374ms step_avg:154.26ms
step:921/1480 train_time:140532ms step_avg:154.26ms
step:922/1480 train_time:140693ms step_avg:154.27ms
step:923/1480 train_time:140850ms step_avg:154.27ms
step:924/1480 train_time:141008ms step_avg:154.28ms
step:925/1480 train_time:141168ms step_avg:154.28ms
step:926/1480 train_time:141327ms step_avg:154.29ms
step:927/1480 train_time:141487ms step_avg:154.29ms
step:928/1480 train_time:141646ms step_avg:154.30ms
step:929/1480 train_time:141806ms step_avg:154.30ms
step:930/1480 train_time:141964ms step_avg:154.31ms
step:931/1480 train_time:142123ms step_avg:154.31ms
step:932/1480 train_time:142281ms step_avg:154.32ms
step:933/1480 train_time:142442ms step_avg:154.32ms
step:934/1480 train_time:142602ms step_avg:154.33ms
step:935/1480 train_time:142766ms step_avg:154.34ms
step:936/1480 train_time:142926ms step_avg:154.35ms
step:937/1480 train_time:143087ms step_avg:154.35ms
step:938/1480 train_time:143244ms step_avg:154.36ms
step:939/1480 train_time:143406ms step_avg:154.37ms
step:940/1480 train_time:143569ms step_avg:154.38ms
step:941/1480 train_time:143728ms step_avg:154.38ms
step:942/1480 train_time:143887ms step_avg:154.38ms
step:943/1480 train_time:144047ms step_avg:154.39ms
step:944/1480 train_time:144209ms step_avg:154.40ms
step:945/1480 train_time:144368ms step_avg:154.40ms
step:946/1480 train_time:144531ms step_avg:154.41ms
step:947/1480 train_time:144692ms step_avg:154.42ms
step:948/1480 train_time:144852ms step_avg:154.43ms
step:949/1480 train_time:145019ms step_avg:154.44ms
step:950/1480 train_time:145170ms step_avg:154.44ms
step:951/1480 train_time:145330ms step_avg:154.44ms
step:952/1480 train_time:145490ms step_avg:154.45ms
step:953/1480 train_time:145650ms step_avg:154.45ms
step:954/1480 train_time:145810ms step_avg:154.46ms
step:955/1480 train_time:145969ms step_avg:154.46ms
step:956/1480 train_time:146128ms step_avg:154.47ms
step:957/1480 train_time:146288ms step_avg:154.48ms
step:958/1480 train_time:146454ms step_avg:154.49ms
step:959/1480 train_time:146611ms step_avg:154.49ms
step:960/1480 train_time:146772ms step_avg:154.50ms
step:961/1480 train_time:146932ms step_avg:154.50ms
step:962/1480 train_time:147089ms step_avg:154.51ms
step:963/1480 train_time:147250ms step_avg:154.51ms
step:964/1480 train_time:147412ms step_avg:154.52ms
step:965/1480 train_time:147571ms step_avg:154.53ms
step:966/1480 train_time:147730ms step_avg:154.53ms
step:967/1480 train_time:147888ms step_avg:154.53ms
step:968/1480 train_time:148047ms step_avg:154.54ms
step:969/1480 train_time:148209ms step_avg:154.54ms
step:970/1480 train_time:148367ms step_avg:154.55ms
step:971/1480 train_time:148525ms step_avg:154.55ms
step:972/1480 train_time:148684ms step_avg:154.56ms
step:973/1480 train_time:148841ms step_avg:154.56ms
step:974/1480 train_time:149001ms step_avg:154.57ms
step:975/1480 train_time:149160ms step_avg:154.57ms
step:976/1480 train_time:149320ms step_avg:154.58ms
step:977/1480 train_time:149478ms step_avg:154.58ms
step:978/1480 train_time:149637ms step_avg:154.58ms
step:979/1480 train_time:149796ms step_avg:154.59ms
step:980/1480 train_time:149955ms step_avg:154.59ms
step:981/1480 train_time:150115ms step_avg:154.60ms
step:982/1480 train_time:150273ms step_avg:154.60ms
step:983/1480 train_time:150435ms step_avg:154.61ms
step:984/1480 train_time:150593ms step_avg:154.61ms
step:985/1480 train_time:150755ms step_avg:154.62ms
step:986/1480 train_time:150913ms step_avg:154.62ms
step:987/1480 train_time:151071ms step_avg:154.63ms
step:988/1480 train_time:151230ms step_avg:154.63ms
step:989/1480 train_time:151388ms step_avg:154.64ms
step:990/1480 train_time:151551ms step_avg:154.64ms
step:991/1480 train_time:151712ms step_avg:154.65ms
step:992/1480 train_time:151875ms step_avg:154.66ms
step:993/1480 train_time:152043ms step_avg:154.67ms
step:994/1480 train_time:152202ms step_avg:154.68ms
step:995/1480 train_time:152361ms step_avg:154.68ms
step:996/1480 train_time:152519ms step_avg:154.69ms
step:997/1480 train_time:152680ms step_avg:154.69ms
step:998/1480 train_time:152841ms step_avg:154.70ms
step:999/1480 train_time:153003ms step_avg:154.70ms
step:1000/1480 train_time:153165ms step_avg:154.71ms
step:1000/1480 val_loss:3.4398 train_time:153237ms step_avg:154.79ms
step:1001/1480 train_time:153335ms step_avg:154.73ms
step:1002/1480 train_time:153489ms step_avg:154.73ms
step:1003/1480 train_time:153653ms step_avg:154.74ms
step:1004/1480 train_time:153816ms step_avg:154.74ms
step:1005/1480 train_time:153976ms step_avg:154.75ms
step:1006/1480 train_time:154136ms step_avg:154.75ms
step:1007/1480 train_time:154296ms step_avg:154.76ms
step:1008/1480 train_time:154457ms step_avg:154.77ms
step:1009/1480 train_time:154622ms step_avg:154.78ms
step:1010/1480 train_time:154780ms step_avg:154.78ms
step:1011/1480 train_time:154939ms step_avg:154.78ms
step:1012/1480 train_time:155097ms step_avg:154.79ms
step:1013/1480 train_time:155258ms step_avg:154.79ms
step:1014/1480 train_time:155417ms step_avg:154.80ms
step:1015/1480 train_time:155580ms step_avg:154.81ms
step:1016/1480 train_time:155741ms step_avg:154.81ms
step:1017/1480 train_time:155902ms step_avg:154.82ms
step:1018/1480 train_time:156062ms step_avg:154.82ms
step:1019/1480 train_time:156223ms step_avg:154.83ms
step:1020/1480 train_time:156382ms step_avg:154.83ms
step:1021/1480 train_time:156542ms step_avg:154.84ms
step:1022/1480 train_time:156701ms step_avg:154.84ms
step:1023/1480 train_time:156862ms step_avg:154.85ms
step:1024/1480 train_time:157022ms step_avg:154.85ms
step:1025/1480 train_time:157182ms step_avg:154.86ms
step:1026/1480 train_time:157342ms step_avg:154.86ms
step:1027/1480 train_time:157501ms step_avg:154.87ms
step:1028/1480 train_time:157663ms step_avg:154.88ms
step:1029/1480 train_time:157827ms step_avg:154.88ms
step:1030/1480 train_time:157988ms step_avg:154.89ms
step:1031/1480 train_time:158147ms step_avg:154.89ms
step:1032/1480 train_time:158313ms step_avg:154.90ms
step:1033/1480 train_time:158473ms step_avg:154.91ms
step:1034/1480 train_time:158636ms step_avg:154.92ms
step:1035/1480 train_time:158795ms step_avg:154.92ms
step:1036/1480 train_time:158955ms step_avg:154.93ms
step:1037/1480 train_time:159115ms step_avg:154.93ms
step:1038/1480 train_time:159275ms step_avg:154.94ms
step:1039/1480 train_time:159438ms step_avg:154.94ms
step:1040/1480 train_time:159598ms step_avg:154.95ms
step:1041/1480 train_time:159758ms step_avg:154.95ms
step:1042/1480 train_time:159917ms step_avg:154.96ms
step:1043/1480 train_time:160075ms step_avg:154.96ms
step:1044/1480 train_time:160233ms step_avg:154.96ms
step:1045/1480 train_time:160394ms step_avg:154.97ms
step:1046/1480 train_time:160556ms step_avg:154.98ms
step:1047/1480 train_time:160717ms step_avg:154.98ms
step:1048/1480 train_time:160877ms step_avg:154.99ms
step:1049/1480 train_time:161037ms step_avg:154.99ms
step:1050/1480 train_time:161198ms step_avg:155.00ms
step:1051/1480 train_time:161359ms step_avg:155.00ms
step:1052/1480 train_time:161522ms step_avg:155.01ms
step:1053/1480 train_time:161682ms step_avg:155.02ms
step:1054/1480 train_time:161844ms step_avg:155.02ms
step:1055/1480 train_time:162004ms step_avg:155.03ms
step:1056/1480 train_time:162163ms step_avg:155.03ms
step:1057/1480 train_time:162324ms step_avg:155.04ms
step:1058/1480 train_time:162489ms step_avg:155.05ms
step:1059/1480 train_time:162653ms step_avg:155.05ms
step:1060/1480 train_time:162815ms step_avg:155.06ms
step:1061/1480 train_time:162973ms step_avg:155.06ms
step:1062/1480 train_time:163132ms step_avg:155.07ms
step:1063/1480 train_time:163291ms step_avg:155.07ms
step:1064/1480 train_time:163450ms step_avg:155.08ms
step:1065/1480 train_time:163613ms step_avg:155.08ms
step:1066/1480 train_time:163775ms step_avg:155.09ms
step:1067/1480 train_time:163938ms step_avg:155.10ms
step:1068/1480 train_time:164099ms step_avg:155.10ms
step:1069/1480 train_time:164262ms step_avg:155.11ms
step:1070/1480 train_time:164421ms step_avg:155.11ms
step:1071/1480 train_time:164583ms step_avg:155.12ms
step:1072/1480 train_time:164742ms step_avg:155.12ms
step:1073/1480 train_time:164900ms step_avg:155.13ms
step:1074/1480 train_time:165059ms step_avg:155.13ms
step:1075/1480 train_time:165220ms step_avg:155.14ms
step:1076/1480 train_time:165378ms step_avg:155.14ms
step:1077/1480 train_time:165539ms step_avg:155.14ms
step:1078/1480 train_time:165703ms step_avg:155.15ms
step:1079/1480 train_time:165866ms step_avg:155.16ms
step:1080/1480 train_time:166027ms step_avg:155.17ms
step:1081/1480 train_time:166186ms step_avg:155.17ms
step:1082/1480 train_time:166346ms step_avg:155.17ms
step:1083/1480 train_time:166506ms step_avg:155.18ms
step:1084/1480 train_time:166666ms step_avg:155.18ms
step:1085/1480 train_time:166827ms step_avg:155.19ms
step:1086/1480 train_time:166989ms step_avg:155.19ms
step:1087/1480 train_time:167149ms step_avg:155.20ms
step:1088/1480 train_time:167311ms step_avg:155.20ms
step:1089/1480 train_time:167476ms step_avg:155.21ms
step:1090/1480 train_time:167638ms step_avg:155.22ms
step:1091/1480 train_time:167799ms step_avg:155.23ms
step:1092/1480 train_time:167960ms step_avg:155.23ms
step:1093/1480 train_time:168122ms step_avg:155.24ms
step:1094/1480 train_time:168280ms step_avg:155.24ms
step:1095/1480 train_time:168441ms step_avg:155.25ms
step:1096/1480 train_time:168603ms step_avg:155.25ms
step:1097/1480 train_time:168764ms step_avg:155.26ms
step:1098/1480 train_time:168927ms step_avg:155.26ms
step:1099/1480 train_time:169088ms step_avg:155.27ms
step:1100/1480 train_time:169253ms step_avg:155.28ms
step:1101/1480 train_time:169417ms step_avg:155.29ms
step:1102/1480 train_time:169579ms step_avg:155.29ms
step:1103/1480 train_time:169745ms step_avg:155.30ms
step:1104/1480 train_time:169906ms step_avg:155.31ms
step:1105/1480 train_time:170070ms step_avg:155.31ms
step:1106/1480 train_time:170232ms step_avg:155.32ms
step:1107/1480 train_time:170394ms step_avg:155.33ms
step:1108/1480 train_time:170554ms step_avg:155.33ms
step:1109/1480 train_time:170714ms step_avg:155.34ms
step:1110/1480 train_time:170877ms step_avg:155.34ms
step:1111/1480 train_time:171037ms step_avg:155.35ms
step:1112/1480 train_time:171201ms step_avg:155.35ms
step:1113/1480 train_time:171371ms step_avg:155.37ms
step:1114/1480 train_time:171533ms step_avg:155.37ms
step:1115/1480 train_time:171696ms step_avg:155.38ms
step:1116/1480 train_time:171857ms step_avg:155.39ms
step:1117/1480 train_time:172020ms step_avg:155.39ms
step:1118/1480 train_time:172184ms step_avg:155.40ms
step:1119/1480 train_time:172346ms step_avg:155.41ms
step:1120/1480 train_time:172508ms step_avg:155.41ms
step:1121/1480 train_time:172671ms step_avg:155.42ms
step:1122/1480 train_time:172830ms step_avg:155.42ms
step:1123/1480 train_time:172991ms step_avg:155.43ms
step:1124/1480 train_time:173155ms step_avg:155.44ms
step:1125/1480 train_time:173317ms step_avg:155.44ms
step:1125/1480 val_loss:3.3839 train_time:173392ms step_avg:155.51ms
step:1126/1480 train_time:173488ms step_avg:155.46ms
step:1127/1480 train_time:173644ms step_avg:155.46ms
step:1128/1480 train_time:173805ms step_avg:155.46ms
step:1129/1480 train_time:173967ms step_avg:155.47ms
step:1130/1480 train_time:174127ms step_avg:155.47ms
step:1131/1480 train_time:174296ms step_avg:155.48ms
step:1132/1480 train_time:174458ms step_avg:155.49ms
step:1133/1480 train_time:174620ms step_avg:155.49ms
step:1134/1480 train_time:174784ms step_avg:155.50ms
step:1135/1480 train_time:174943ms step_avg:155.51ms
step:1136/1480 train_time:175105ms step_avg:155.51ms
step:1137/1480 train_time:175266ms step_avg:155.52ms
step:1138/1480 train_time:175431ms step_avg:155.52ms
step:1139/1480 train_time:175599ms step_avg:155.54ms
step:1140/1480 train_time:175753ms step_avg:155.53ms
step:1141/1480 train_time:175917ms step_avg:155.54ms
step:1142/1480 train_time:176079ms step_avg:155.55ms
step:1143/1480 train_time:176241ms step_avg:155.55ms
step:1144/1480 train_time:176404ms step_avg:155.56ms
step:1145/1480 train_time:176563ms step_avg:155.56ms
step:1146/1480 train_time:176726ms step_avg:155.57ms
step:1147/1480 train_time:176886ms step_avg:155.57ms
step:1148/1480 train_time:177046ms step_avg:155.58ms
step:1149/1480 train_time:177208ms step_avg:155.58ms
step:1150/1480 train_time:177368ms step_avg:155.59ms
step:1151/1480 train_time:177533ms step_avg:155.59ms
step:1152/1480 train_time:177698ms step_avg:155.60ms
step:1153/1480 train_time:177863ms step_avg:155.61ms
step:1154/1480 train_time:178023ms step_avg:155.61ms
step:1155/1480 train_time:178184ms step_avg:155.62ms
step:1156/1480 train_time:178351ms step_avg:155.63ms
step:1157/1480 train_time:178514ms step_avg:155.64ms
step:1158/1480 train_time:178676ms step_avg:155.64ms
step:1159/1480 train_time:178839ms step_avg:155.65ms
step:1160/1480 train_time:179000ms step_avg:155.65ms
step:1161/1480 train_time:179163ms step_avg:155.66ms
step:1162/1480 train_time:179326ms step_avg:155.66ms
step:1163/1480 train_time:179487ms step_avg:155.67ms
step:1164/1480 train_time:179647ms step_avg:155.67ms
step:1165/1480 train_time:179807ms step_avg:155.68ms
step:1166/1480 train_time:179969ms step_avg:155.68ms
step:1167/1480 train_time:180129ms step_avg:155.69ms
step:1168/1480 train_time:180291ms step_avg:155.69ms
step:1169/1480 train_time:180455ms step_avg:155.70ms
step:1170/1480 train_time:180615ms step_avg:155.70ms
step:1171/1480 train_time:180777ms step_avg:155.71ms
step:1172/1480 train_time:180937ms step_avg:155.71ms
step:1173/1480 train_time:181099ms step_avg:155.72ms
step:1174/1480 train_time:181268ms step_avg:155.73ms
step:1175/1480 train_time:181431ms step_avg:155.74ms
step:1176/1480 train_time:181595ms step_avg:155.74ms
step:1177/1480 train_time:181763ms step_avg:155.75ms
step:1178/1480 train_time:181924ms step_avg:155.76ms
step:1179/1480 train_time:182083ms step_avg:155.76ms
step:1180/1480 train_time:182251ms step_avg:155.77ms
step:1181/1480 train_time:182412ms step_avg:155.77ms
step:1182/1480 train_time:182573ms step_avg:155.78ms
step:1183/1480 train_time:182736ms step_avg:155.79ms
step:1184/1480 train_time:182899ms step_avg:155.79ms
step:1185/1480 train_time:183063ms step_avg:155.80ms
step:1186/1480 train_time:183225ms step_avg:155.80ms
step:1187/1480 train_time:183396ms step_avg:155.82ms
step:1188/1480 train_time:183556ms step_avg:155.82ms
step:1189/1480 train_time:183718ms step_avg:155.83ms
step:1190/1480 train_time:183880ms step_avg:155.83ms
step:1191/1480 train_time:184043ms step_avg:155.84ms
step:1192/1480 train_time:184204ms step_avg:155.84ms
step:1193/1480 train_time:184364ms step_avg:155.84ms
step:1194/1480 train_time:184525ms step_avg:155.85ms
step:1195/1480 train_time:184687ms step_avg:155.85ms
step:1196/1480 train_time:184859ms step_avg:155.87ms
step:1197/1480 train_time:185020ms step_avg:155.87ms
step:1198/1480 train_time:185188ms step_avg:155.88ms
step:1199/1480 train_time:185351ms step_avg:155.89ms
step:1200/1480 train_time:185513ms step_avg:155.89ms
step:1201/1480 train_time:185674ms step_avg:155.90ms
step:1202/1480 train_time:185843ms step_avg:155.91ms
step:1203/1480 train_time:186008ms step_avg:155.92ms
step:1204/1480 train_time:186172ms step_avg:155.92ms
step:1205/1480 train_time:186333ms step_avg:155.93ms
step:1206/1480 train_time:186495ms step_avg:155.93ms
step:1207/1480 train_time:186655ms step_avg:155.94ms
step:1208/1480 train_time:186817ms step_avg:155.94ms
step:1209/1480 train_time:186982ms step_avg:155.95ms
step:1210/1480 train_time:187147ms step_avg:155.96ms
step:1211/1480 train_time:187310ms step_avg:155.96ms
step:1212/1480 train_time:187474ms step_avg:155.97ms
step:1213/1480 train_time:187640ms step_avg:155.98ms
step:1214/1480 train_time:187807ms step_avg:155.99ms
step:1215/1480 train_time:187969ms step_avg:155.99ms
step:1216/1480 train_time:188129ms step_avg:155.99ms
step:1217/1480 train_time:188293ms step_avg:156.00ms
step:1218/1480 train_time:188455ms step_avg:156.01ms
step:1219/1480 train_time:188623ms step_avg:156.02ms
step:1220/1480 train_time:188786ms step_avg:156.02ms
step:1221/1480 train_time:188947ms step_avg:156.03ms
step:1222/1480 train_time:189108ms step_avg:156.03ms
step:1223/1480 train_time:189271ms step_avg:156.04ms
step:1224/1480 train_time:189440ms step_avg:156.05ms
step:1225/1480 train_time:189605ms step_avg:156.05ms
step:1226/1480 train_time:189769ms step_avg:156.06ms
step:1227/1480 train_time:189935ms step_avg:156.07ms
step:1228/1480 train_time:190098ms step_avg:156.07ms
step:1229/1480 train_time:190262ms step_avg:156.08ms
step:1230/1480 train_time:190429ms step_avg:156.09ms
step:1231/1480 train_time:190595ms step_avg:156.10ms
step:1232/1480 train_time:190761ms step_avg:156.11ms
step:1233/1480 train_time:190923ms step_avg:156.11ms
step:1234/1480 train_time:191085ms step_avg:156.12ms
step:1235/1480 train_time:191249ms step_avg:156.12ms
step:1236/1480 train_time:191410ms step_avg:156.13ms
step:1237/1480 train_time:191570ms step_avg:156.13ms
step:1238/1480 train_time:191745ms step_avg:156.14ms
step:1239/1480 train_time:191907ms step_avg:156.15ms
step:1240/1480 train_time:192071ms step_avg:156.15ms
step:1241/1480 train_time:192237ms step_avg:156.16ms
step:1242/1480 train_time:192399ms step_avg:156.17ms
step:1243/1480 train_time:192563ms step_avg:156.17ms
step:1244/1480 train_time:192724ms step_avg:156.18ms
step:1245/1480 train_time:192886ms step_avg:156.18ms
step:1246/1480 train_time:193046ms step_avg:156.19ms
step:1247/1480 train_time:193210ms step_avg:156.19ms
step:1248/1480 train_time:193373ms step_avg:156.20ms
step:1249/1480 train_time:193534ms step_avg:156.20ms
step:1250/1480 train_time:193699ms step_avg:156.21ms
step:1250/1480 val_loss:3.3341 train_time:193774ms step_avg:156.27ms
step:1251/1480 train_time:193871ms step_avg:156.22ms
step:1252/1480 train_time:194028ms step_avg:156.22ms
step:1253/1480 train_time:194190ms step_avg:156.23ms
step:1254/1480 train_time:194351ms step_avg:156.23ms
step:1255/1480 train_time:194521ms step_avg:156.24ms
step:1256/1480 train_time:194685ms step_avg:156.25ms
step:1257/1480 train_time:194847ms step_avg:156.25ms
step:1258/1480 train_time:195013ms step_avg:156.26ms
step:1259/1480 train_time:195176ms step_avg:156.27ms
step:1260/1480 train_time:195336ms step_avg:156.27ms
step:1261/1480 train_time:195499ms step_avg:156.27ms
step:1262/1480 train_time:195665ms step_avg:156.28ms
step:1263/1480 train_time:195830ms step_avg:156.29ms
step:1264/1480 train_time:195989ms step_avg:156.29ms
step:1265/1480 train_time:196149ms step_avg:156.29ms
step:1266/1480 train_time:196310ms step_avg:156.30ms
step:1267/1480 train_time:196471ms step_avg:156.30ms
step:1268/1480 train_time:196634ms step_avg:156.31ms
step:1269/1480 train_time:196800ms step_avg:156.31ms
step:1270/1480 train_time:196963ms step_avg:156.32ms
step:1271/1480 train_time:197127ms step_avg:156.33ms
step:1272/1480 train_time:197287ms step_avg:156.33ms
step:1273/1480 train_time:197452ms step_avg:156.34ms
step:1274/1480 train_time:197616ms step_avg:156.34ms
step:1275/1480 train_time:197777ms step_avg:156.35ms
step:1276/1480 train_time:197937ms step_avg:156.35ms
step:1277/1480 train_time:198100ms step_avg:156.35ms
step:1278/1480 train_time:198260ms step_avg:156.36ms
step:1279/1480 train_time:198424ms step_avg:156.36ms
step:1280/1480 train_time:198589ms step_avg:156.37ms
step:1281/1480 train_time:198751ms step_avg:156.37ms
step:1282/1480 train_time:198911ms step_avg:156.38ms
step:1283/1480 train_time:199073ms step_avg:156.38ms
step:1284/1480 train_time:199237ms step_avg:156.39ms
step:1285/1480 train_time:199398ms step_avg:156.39ms
step:1286/1480 train_time:199562ms step_avg:156.40ms
step:1287/1480 train_time:199726ms step_avg:156.40ms
step:1288/1480 train_time:199888ms step_avg:156.41ms
step:1289/1480 train_time:200056ms step_avg:156.42ms
step:1290/1480 train_time:200226ms step_avg:156.43ms
step:1291/1480 train_time:200390ms step_avg:156.43ms
step:1292/1480 train_time:200553ms step_avg:156.44ms
step:1293/1480 train_time:200720ms step_avg:156.45ms
step:1294/1480 train_time:200885ms step_avg:156.45ms
step:1295/1480 train_time:201049ms step_avg:156.46ms
step:1296/1480 train_time:201211ms step_avg:156.46ms
step:1297/1480 train_time:201376ms step_avg:156.47ms
step:1298/1480 train_time:201540ms step_avg:156.48ms
step:1299/1480 train_time:201704ms step_avg:156.48ms
step:1300/1480 train_time:201865ms step_avg:156.48ms
step:1301/1480 train_time:202026ms step_avg:156.49ms
step:1302/1480 train_time:202192ms step_avg:156.50ms
step:1303/1480 train_time:202357ms step_avg:156.50ms
step:1304/1480 train_time:202524ms step_avg:156.51ms
step:1305/1480 train_time:202687ms step_avg:156.52ms
step:1306/1480 train_time:202852ms step_avg:156.52ms
step:1307/1480 train_time:203012ms step_avg:156.52ms
step:1308/1480 train_time:203172ms step_avg:156.53ms
step:1309/1480 train_time:203336ms step_avg:156.53ms
step:1310/1480 train_time:203498ms step_avg:156.54ms
step:1311/1480 train_time:203659ms step_avg:156.54ms
step:1312/1480 train_time:203825ms step_avg:156.55ms
step:1313/1480 train_time:203988ms step_avg:156.55ms
step:1314/1480 train_time:204155ms step_avg:156.56ms
step:1315/1480 train_time:204317ms step_avg:156.57ms
step:1316/1480 train_time:204477ms step_avg:156.57ms
step:1317/1480 train_time:204641ms step_avg:156.57ms
step:1318/1480 train_time:204808ms step_avg:156.58ms
step:1319/1480 train_time:204975ms step_avg:156.59ms
step:1320/1480 train_time:205142ms step_avg:156.60ms
step:1321/1480 train_time:205307ms step_avg:156.60ms
step:1322/1480 train_time:205477ms step_avg:156.61ms
step:1323/1480 train_time:205641ms step_avg:156.62ms
step:1324/1480 train_time:205806ms step_avg:156.63ms
step:1325/1480 train_time:205976ms step_avg:156.64ms
step:1326/1480 train_time:206142ms step_avg:156.64ms
step:1327/1480 train_time:206306ms step_avg:156.65ms
step:1328/1480 train_time:206468ms step_avg:156.65ms
step:1329/1480 train_time:206650ms step_avg:156.67ms
step:1330/1480 train_time:206816ms step_avg:156.68ms
step:1331/1480 train_time:206978ms step_avg:156.68ms
step:1332/1480 train_time:207143ms step_avg:156.69ms
step:1333/1480 train_time:207308ms step_avg:156.70ms
step:1334/1480 train_time:207471ms step_avg:156.70ms
step:1335/1480 train_time:207632ms step_avg:156.70ms
step:1336/1480 train_time:207802ms step_avg:156.71ms
step:1337/1480 train_time:207969ms step_avg:156.72ms
step:1338/1480 train_time:208134ms step_avg:156.73ms
step:1339/1480 train_time:208296ms step_avg:156.73ms
step:1340/1480 train_time:208459ms step_avg:156.74ms
step:1341/1480 train_time:208620ms step_avg:156.74ms
step:1342/1480 train_time:208787ms step_avg:156.75ms
step:1343/1480 train_time:208949ms step_avg:156.75ms
step:1344/1480 train_time:209111ms step_avg:156.75ms
step:1345/1480 train_time:209278ms step_avg:156.76ms
step:1346/1480 train_time:209441ms step_avg:156.77ms
step:1347/1480 train_time:209604ms step_avg:156.77ms
step:1348/1480 train_time:209767ms step_avg:156.78ms
step:1349/1480 train_time:209928ms step_avg:156.78ms
step:1350/1480 train_time:210093ms step_avg:156.79ms
step:1351/1480 train_time:210254ms step_avg:156.79ms
step:1352/1480 train_time:210416ms step_avg:156.79ms
step:1353/1480 train_time:210583ms step_avg:156.80ms
step:1354/1480 train_time:210747ms step_avg:156.81ms
step:1355/1480 train_time:210908ms step_avg:156.81ms
step:1356/1480 train_time:211072ms step_avg:156.81ms
step:1357/1480 train_time:211235ms step_avg:156.82ms
step:1358/1480 train_time:211401ms step_avg:156.83ms
step:1359/1480 train_time:211564ms step_avg:156.83ms
step:1360/1480 train_time:211730ms step_avg:156.84ms
step:1361/1480 train_time:211896ms step_avg:156.84ms
step:1362/1480 train_time:212061ms step_avg:156.85ms
step:1363/1480 train_time:212230ms step_avg:156.86ms
step:1364/1480 train_time:212391ms step_avg:156.86ms
step:1365/1480 train_time:212552ms step_avg:156.86ms
step:1366/1480 train_time:212716ms step_avg:156.87ms
step:1367/1480 train_time:212880ms step_avg:156.88ms
step:1368/1480 train_time:213045ms step_avg:156.88ms
step:1369/1480 train_time:213214ms step_avg:156.89ms
step:1370/1480 train_time:213380ms step_avg:156.90ms
step:1371/1480 train_time:213543ms step_avg:156.90ms
step:1372/1480 train_time:213711ms step_avg:156.91ms
step:1373/1480 train_time:213871ms step_avg:156.91ms
step:1374/1480 train_time:214039ms step_avg:156.92ms
step:1375/1480 train_time:214202ms step_avg:156.92ms
step:1375/1480 val_loss:3.2957 train_time:214277ms step_avg:156.98ms
step:1376/1480 train_time:214375ms step_avg:156.94ms
step:1377/1480 train_time:214533ms step_avg:156.94ms
step:1378/1480 train_time:214696ms step_avg:156.94ms
step:1379/1480 train_time:214861ms step_avg:156.95ms
step:1380/1480 train_time:215023ms step_avg:156.95ms
step:1381/1480 train_time:215193ms step_avg:156.96ms
step:1382/1480 train_time:215357ms step_avg:156.97ms
step:1383/1480 train_time:215519ms step_avg:156.97ms
step:1384/1480 train_time:215686ms step_avg:156.98ms
step:1385/1480 train_time:215847ms step_avg:156.98ms
step:1386/1480 train_time:216009ms step_avg:156.98ms
step:1387/1480 train_time:216174ms step_avg:156.99ms
step:1388/1480 train_time:216334ms step_avg:156.99ms
step:1389/1480 train_time:216499ms step_avg:157.00ms
step:1390/1480 train_time:216660ms step_avg:157.00ms
step:1391/1480 train_time:216822ms step_avg:157.00ms
step:1392/1480 train_time:216988ms step_avg:157.01ms
step:1393/1480 train_time:217151ms step_avg:157.01ms
step:1394/1480 train_time:217315ms step_avg:157.02ms
step:1395/1480 train_time:217478ms step_avg:157.02ms
step:1396/1480 train_time:217641ms step_avg:157.03ms
step:1397/1480 train_time:217800ms step_avg:157.03ms
step:1398/1480 train_time:217962ms step_avg:157.03ms
step:1399/1480 train_time:218123ms step_avg:157.04ms
step:1400/1480 train_time:218292ms step_avg:157.04ms
step:1401/1480 train_time:218453ms step_avg:157.05ms
step:1402/1480 train_time:218615ms step_avg:157.05ms
step:1403/1480 train_time:218781ms step_avg:157.06ms
step:1404/1480 train_time:218943ms step_avg:157.06ms
step:1405/1480 train_time:219109ms step_avg:157.07ms
step:1406/1480 train_time:219275ms step_avg:157.07ms
step:1407/1480 train_time:219436ms step_avg:157.08ms
step:1408/1480 train_time:219597ms step_avg:157.08ms
step:1409/1480 train_time:219770ms step_avg:157.09ms
step:1410/1480 train_time:219932ms step_avg:157.09ms
step:1411/1480 train_time:220092ms step_avg:157.10ms
step:1412/1480 train_time:220253ms step_avg:157.10ms
step:1413/1480 train_time:220416ms step_avg:157.10ms
step:1414/1480 train_time:220581ms step_avg:157.11ms
step:1415/1480 train_time:220747ms step_avg:157.12ms
step:1416/1480 train_time:220920ms step_avg:157.13ms
step:1417/1480 train_time:221084ms step_avg:157.13ms
step:1418/1480 train_time:221249ms step_avg:157.14ms
step:1419/1480 train_time:221413ms step_avg:157.14ms
step:1420/1480 train_time:221577ms step_avg:157.15ms
step:1421/1480 train_time:221742ms step_avg:157.15ms
step:1422/1480 train_time:221908ms step_avg:157.16ms
step:1423/1480 train_time:222072ms step_avg:157.16ms
step:1424/1480 train_time:222236ms step_avg:157.17ms
step:1425/1480 train_time:222406ms step_avg:157.18ms
step:1426/1480 train_time:222571ms step_avg:157.18ms
step:1427/1480 train_time:222736ms step_avg:157.19ms
step:1428/1480 train_time:222898ms step_avg:157.19ms
step:1429/1480 train_time:223059ms step_avg:157.19ms
step:1430/1480 train_time:223222ms step_avg:157.20ms
step:1431/1480 train_time:223388ms step_avg:157.20ms
step:1432/1480 train_time:223557ms step_avg:157.21ms
step:1433/1480 train_time:223725ms step_avg:157.22ms
step:1434/1480 train_time:223894ms step_avg:157.23ms
step:1435/1480 train_time:224059ms step_avg:157.23ms
step:1436/1480 train_time:224226ms step_avg:157.24ms
step:1437/1480 train_time:224390ms step_avg:157.25ms
step:1438/1480 train_time:224552ms step_avg:157.25ms
step:1439/1480 train_time:224718ms step_avg:157.26ms
step:1440/1480 train_time:224880ms step_avg:157.26ms
step:1441/1480 train_time:225046ms step_avg:157.26ms
step:1442/1480 train_time:225213ms step_avg:157.27ms
step:1443/1480 train_time:225386ms step_avg:157.28ms
step:1444/1480 train_time:225551ms step_avg:157.29ms
step:1445/1480 train_time:225713ms step_avg:157.29ms
step:1446/1480 train_time:225879ms step_avg:157.30ms
step:1447/1480 train_time:226049ms step_avg:157.31ms
step:1448/1480 train_time:226211ms step_avg:157.31ms
step:1449/1480 train_time:226375ms step_avg:157.31ms
step:1450/1480 train_time:226538ms step_avg:157.32ms
step:1451/1480 train_time:226702ms step_avg:157.32ms
step:1452/1480 train_time:226870ms step_avg:157.33ms
step:1453/1480 train_time:227033ms step_avg:157.33ms
step:1454/1480 train_time:227196ms step_avg:157.34ms
step:1455/1480 train_time:227362ms step_avg:157.34ms
step:1456/1480 train_time:227526ms step_avg:157.35ms
step:1457/1480 train_time:227690ms step_avg:157.35ms
step:1458/1480 train_time:227853ms step_avg:157.36ms
step:1459/1480 train_time:228018ms step_avg:157.36ms
step:1460/1480 train_time:228181ms step_avg:157.37ms
step:1461/1480 train_time:228344ms step_avg:157.37ms
step:1462/1480 train_time:228509ms step_avg:157.38ms
step:1463/1480 train_time:228676ms step_avg:157.38ms
step:1464/1480 train_time:228839ms step_avg:157.39ms
step:1465/1480 train_time:229003ms step_avg:157.39ms
step:1466/1480 train_time:229166ms step_avg:157.39ms
step:1467/1480 train_time:229332ms step_avg:157.40ms
step:1468/1480 train_time:229496ms step_avg:157.40ms
step:1469/1480 train_time:229658ms step_avg:157.41ms
step:1470/1480 train_time:229826ms step_avg:157.42ms
step:1471/1480 train_time:229999ms step_avg:157.43ms
step:1472/1480 train_time:230171ms step_avg:157.44ms
step:1473/1480 train_time:230335ms step_avg:157.44ms
step:1474/1480 train_time:230500ms step_avg:157.45ms
step:1475/1480 train_time:230670ms step_avg:157.45ms
step:1476/1480 train_time:230834ms step_avg:157.46ms
step:1477/1480 train_time:231000ms step_avg:157.46ms
step:1478/1480 train_time:231172ms step_avg:157.47ms
step:1479/1480 train_time:231336ms step_avg:157.48ms
step:1480/1480 train_time:231498ms step_avg:157.48ms
step:1480/1480 val_loss:3.2767 train_time:231574ms step_avg:157.53ms
peak memory consumption: 34239 MiB