eca39347-6616-4903-ad6b-bab828aa2f78.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
import contextlib
from dataclasses import dataclass
from pathlib import Path

import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.nn.attention.flex_attention import BlockMask, flex_attention #KoszarskyB

# -----------------------------------------------------------------------------
# Muon optimizer

@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert len(G.shape) == 2
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    X /= (X.norm() + eps) # ensure top singular value <= 1
    if G.size(0) > G.size(1):
        X = X.T
    for _ in range(steps):
        A = X @ X.T
        B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    if G.size(0) > G.size(1):
        X = X.T
    return X

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven't tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        ns_steps: The number of Newton-Schulz iteration steps to use.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True, ns_steps=5):
        self.world_size = int(os.environ['WORLD_SIZE'])
        self.rank = int(os.environ['RANK'])
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, ns_steps=ns_steps)
        params = list(params)
        assert all(isinstance(p, torch.Tensor) for p in params)
        sizes = {p.numel() for p in params}
        param_groups = [
            {
                'params': [p for p in params if p.numel() == size],
                'update_buffer': [
                    torch.empty(size, device='cuda', dtype=torch.bfloat16)
                    for _ in range(self.world_size)
                ],
            }
            for size in sizes
        ]
        super().__init__(param_groups, defaults)

    def step(self):

        for group in self.param_groups:

            lr = group['lr']
            momentum = group['momentum']
            nesterov = group['nesterov']
            ns_steps = group['ns_steps']
            update_buffers = group['update_buffer']
            # generate weight updates in distributed fashion
            params = group['params']
            assert len(params) % self.world_size == 0
            handle = None
            params_world = None
            def update_prev():
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffers):
                    p_world.data.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.world_size]:
                p = params[base_i + self.rank]
                g = p.grad
                assert g is not None
                state = self.state[p]
                if 'momentum_buffer' not in state:
                    state['momentum_buffer'] = torch.zeros_like(g)
                buf = state['momentum_buffer']
                buf.lerp_(g, 1 - momentum)
                g = g.lerp_(buf, momentum) if nesterov else buf
                g = zeropower_via_newtonschulz5(g, steps=ns_steps).flatten()
                update_prev()
                handle = dist.all_gather(update_buffers, g, async_op=True)
                params_world = params[base_i : base_i + self.world_size]
            update_prev()

# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):

    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=False)

    def forward(self, x):
        return F.linear(x, self.weight.to(x.dtype))

class Rotary(torch.nn.Module):

    def __init__(self, dim, base=10000):
        super().__init__()
        self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None

    def forward(self, x):
        seq_len = x.shape[1]
        if seq_len != self.seq_len_cached:
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.outer(t, self.inv_freq)
            self.seq_len_cached = seq_len
            self.cos_cached = freqs.cos()
            self.sin_cached = freqs.sin()
        cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
        # apply_rotary_emb(x, cos, sin)
        x1, x2 = x.chunk(2, dim=3)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x)

class CausalSelfAttention(nn.Module):

    def __init__(self, dim, num_heads):
        super().__init__()
        assert dim % num_heads == 0
        self.num_heads = num_heads
        self.c_q = CastedLinear(dim, dim)
        self.c_k = CastedLinear(dim, dim)
        self.c_v = CastedLinear(dim, dim)
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5]))
        self.rotary = Rotary(dim // num_heads) # dim // num_heads = head_dim
        self.c_proj = CastedLinear(dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x, vi, block_mask):
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q = self.c_q(x).view(B, T, self.num_heads, -1)
        k = self.c_k(x).view(B, T, self.num_heads, -1)
        v = self.c_v(x).view(B, T, self.num_heads, -1)
        v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @KoszarskyB & @Grad62304977
        q, k = norm(q), norm(k) # QK norm @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask, enable_gqa=True)
        y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):

    def __init__(self, dim):
        super().__init__()
        self.c_fc   = CastedLinear(dim, 4 * dim)
        self.c_proj = CastedLinear(4 * dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x):
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):

    def __init__(self, config):
        super().__init__()
        self.attn = CausalSelfAttention(config.model_dim, config.num_heads)
        self.mlp = MLP(config.model_dim)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x, vi, x0, block_mask):
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        x = x + self.attn(norm(x), vi, block_mask)
        x = x + self.mlp(norm(x))
        return x

class ValueEmbedding(nn.Module):
    def __init__(self, config: "GPTConfig"):
        super().__init__()
        self.__setattr__
        self.embed = nn.ModuleList([
            nn.Embedding(config.vocab_size, config.model_dim)
            for _ in range(6)
        ])

    def forward(self, inputs) -> "list[torch.Tensor]":
        ve = [emb(inputs) for emb in self.embed]
        ve += reversed(ve)
        return ve


# -----------------------------------------------------------------------------
# The main GPT-2 model

@dataclass
class GPTConfig:
    vocab_size : int = 50304
    num_layers : int = 12
    num_heads : int = 6 # head dim 128 suggested by @Grad62304977
    model_dim : int = 768

class GPT(nn.Module):

    def __init__(self, config: GPTConfig):
        super().__init__()
        self.num_layers = config.num_layers

        # U-net design by @brendanh0gan
        self.num_encoder_layers = config.num_layers // 2 # Half of the layers for encoder
        self.num_decoder_layers = config.num_layers - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))

        self.embed = nn.Embedding(config.vocab_size, config.model_dim)
        self.blocks = nn.ModuleList([Block(config) for _ in range(config.num_layers)])
        # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
        # U-net structure on token value embeddings by @leloykun
        self.value_embeds = ValueEmbedding(config)
        self.lm_head = CastedLinear(config.model_dim, config.vocab_size)
        self.lm_head.weight.data.zero_() # @Grad62304977

    def forward(
        self,
        inputs: torch.Tensor,
        targets: torch.Tensor,
        sliding_window_num_blocks: torch.Tensor,
    ):
        BLOCK_SIZE = 128
        assert inputs.ndim == 1
        docs = (inputs == 50256).cumsum(0)
        docs_low = docs.view(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.view(-1, BLOCK_SIZE)[:, -1].contiguous()

        def document_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            return causal_mask & document_mask

        def dense_to_ordered(dense_mask: torch.Tensor):
            num_blocks = dense_mask.sum(dim=-1, dtype=torch.int32)
            indices = dense_mask.argsort(dim=-1, descending=True, stable=True).to(torch.int32)
            return num_blocks[None, None].contiguous(), indices[None, None].contiguous()

        def create_doc_swc_block_mask(sliding_window_num_blocks: torch.Tensor):
            kv_idx = block_idx = torch.arange(512, dtype=torch.int32, device="cuda")
            q_idx = block_idx[:, None]
            causal_bm = q_idx >= kv_idx
            causal_full_bm = q_idx > kv_idx
            window_bm = q_idx - kv_idx < sliding_window_num_blocks
            window_full_bm = window_bm
            # document_bm = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
            document_bm = (docs_low[:, None] <= docs_high) & (docs_low <= docs_high[:, None])
            document_full_bm = (docs_low[:, None] == docs_high) & (docs_low == docs_high[:, None])
            nonzero_bm = causal_bm & window_bm & document_bm
            full_bm  = causal_full_bm & window_full_bm & document_full_bm
            kv_num_blocks, kv_indices = dense_to_ordered(nonzero_bm ^ full_bm)
            full_kv_num_blocks, full_kv_indices = dense_to_ordered(full_bm)
            return BlockMask.from_kv_blocks(
                kv_num_blocks,
                kv_indices,
                full_kv_num_blocks,
                full_kv_indices,
                BLOCK_SIZE=BLOCK_SIZE,
                mask_mod=document_causal,
            )

        block_mask = create_doc_swc_block_mask(sliding_window_num_blocks)

        # forward the GPT model itself
        x = self.embed(inputs[None]) # token embeddings of shape (b, t, model_dim)
        x = norm(x) # @Grad62304977
        x0 = x
        ve = self.value_embeds(inputs)
        ve_enc, ve_dec = ve[:self.num_encoder_layers], ve[self.num_encoder_layers:]

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        for i in range(self.num_encoder_layers):
            x = self.blocks[i](x, ve_enc[i], x0, block_mask)
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            # U-net structure on token value embeddings by @leloykun
            x = self.blocks[self.num_encoder_layers + i](x, ve_dec[i], x0, block_mask)

        x = norm(x)
        logits = self.lm_head(x)
        logits = 30 * torch.tanh(logits / 30) # @Grad62304977
        logits = logits.float()
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1))
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _peek_data_shard(file: Path):
    # only reads the header, returns header data
    # header is 256 int32
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    return int(header[2]) # number of tokens (claimed)

def _load_data_shard(path: Path, num_tokens):
    with path.open("rb", buffering=0) as f:
        tokens = torch.empty(num_tokens, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy())
        assert nbytes == 2 * num_tokens, "number of tokens read does not match header?"
    return tokens

class DistributedDataLoader:
    def __init__(self, filename_pattern, seq_len, process_rank, num_processes):
        self.process_rank = process_rank
        self.num_processes = num_processes
        self.seq_len = seq_len

        # glob files that match the pattern
        self.files = sorted(Path.cwd().glob(filename_pattern))
        assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"

        # load and validate all data shards, count number of tokens in total
        self.files_num_tokens = [_peek_data_shard(file) for file in self.files]
        assert min(self.files_num_tokens) >= num_processes * seq_len + 1
        self.total_num_tokens = sum(self.files_num_tokens)

        self.reset()

    def reset(self):
        self.current_shard = -1
        self.advance()

    def advance(self): # advance to next data shard
        self.current_shard = (self.current_shard + 1) % len(self.files)
        self.current_position = self.process_rank * self.seq_len
        self.tokens = _load_data_shard(self.files[self.current_shard], self.files_num_tokens[self.current_shard])

    def next_batch(self):
        batch_size = self.seq_len * self.num_processes
        buf = self.tokens[self.current_position:self.current_position+self.seq_len+1]
        # host side async is sufficient;
        # no performance improvement was observed when introducing a separate stream.
        inputs = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
        targets = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
        # advance current position and load next shard if necessary
        self.current_position += batch_size
        if self.current_position + batch_size + 1 >= len(self.tokens):
            self.advance()
        return inputs, targets

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data hyperparams
    input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
    input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
    # optimization hyperparams
    batch_size : int = 8 # batch size, in sequences, across all devices
    sequence_length : int = 64*1024 # sequence length, in tokens
    num_iterations : int = 1480 # number of iterations to run
    warmup_iters : int = 0
    cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
    weight_decay : float = 0
    # evaluation and logging hyperparams
    val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
    val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()

# set up DDP (distributed data parallel). torchrun sets this env variable
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
assert torch.cuda.is_available()
device = torch.device(f"cuda:{ddp_local_rank}")
torch.cuda.set_device(device)
print(f"using device: {device}")
dist.init_process_group(backend='nccl', device_id=device)
dist.barrier()
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = uuid.uuid4()
    logdir = Path("logs") / f"{run_id}"
    logdir.mkdir(exist_ok=True)
    logfile = Path("logs") / f"{run_id}.txt"
    print(logfile.stem)
    # create the log file
    with logfile.open("w") as f:
        # begin the log by printing this file (the Python code)
        print(code, file=f)
        print("=" * 100, file=f)
def print0(s, logonly=False):
    if master_process:
        with logfile.open("a") as f:
            if not logonly:
                print(s)
            print(s, file=f)
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running python {sys.version}")
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)

# calculate the number of steps to take in the val loop.
assert args.val_tokens % (args.sequence_length * ddp_world_size) == 0
val_steps = args.val_tokens // (args.sequence_length * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size

# load tokens
train_loader = DistributedDataLoader(args.input_bin, args.sequence_length, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, args.sequence_length, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.total_num_tokens} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.total_num_tokens} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
inputs_train, targets_train = train_loader.next_batch()

# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, num_layers=12, num_heads=6, model_dim=768))
model = model.cuda().bfloat16()
for m in model.modules():
    if isinstance(m, CastedLinear):
        m.float()
config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank], broadcast_buffers=False, gradient_as_bucket_view=True)
raw_model = model.module # always contains the "raw" unwrapped model

# init the optimizer(s)
embed_params = [*raw_model.embed.parameters(), *raw_model.value_embeds.parameters()]
optimizer1 = torch.optim.Adam(embed_params, lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.blocks.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
    assert it <= args.num_iterations
    # 1) linear warmup for warmup_iters steps
    if it < args.warmup_iters:
        return (it+1) / args.warmup_iters
    # 2) constant lr for a while
    elif it < args.num_iterations - args.cooldown_iters:
        return 1.0
    # 3) linear cooldown
    else:
        decay_ratio = (args.num_iterations - it) / args.cooldown_iters
        return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]

sliding_window_num_blocks = torch.tensor(1, dtype=torch.int32, device="cuda")
sw_num_blocks_prev = 1
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
    last_step = (step == args.num_iterations)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Linearly increase the sliding window size over training in chunks of 64 from 64 -> 1792. By @fernbear.bsky.social
    frac_done = step / args.num_iterations # training progress
    sw_num_blocks = int(((1 - frac_done) * 64 + frac_done * 1792 + 64) // 128)
    if sw_num_blocks != sw_num_blocks_prev:
        sliding_window_num_blocks.copy_(sw_num_blocks, non_blocking=True)
        sw_num_blocks_prev = sw_num_blocks

    # once in a while evaluate the validation dataset
    if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # run validation batches
        model.eval()
        val_loader.reset()
        val_loss = 0.0
        for _ in range(val_steps):
            with torch.no_grad():
                inputs_val, targets_val = val_loader.next_batch()
                val_loss += model(inputs_val, targets_val, sliding_window_num_blocks)
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        val_loss /= val_steps
        # log val loss to console and to logfile
        print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # save the state of the training process
        log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
        torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    # bit confusing: we want to make sure to eval on 0th iteration
    # but also after the very last iteration. so we loop for step <= num_iterations
    # instead of just < num_iterations (one extra due to <=), only to do
    # the validation/sampling one last time, and then we break right here as we're done.
    if last_step:
        break

    # --------------- TRAINING SECTION BEGIN -----------------
    model.train()
    for i in range(1, train_accumulation_steps + 1):
        with contextlib.ExitStack() as stack:
            if i < train_accumulation_steps: # there's no need to sync gradients every accumulation step
                stack.enter_context(model.no_sync())
            if step >= 5:
                stack.enter_context(torch.compiler.set_stance(skip_guard_eval_unsafe=True))
            model(inputs_train, targets_train, sliding_window_num_blocks).backward()
            inputs_train, targets_train = train_loader.next_batch()
    if train_accumulation_steps != 1:
        for p in model.parameters():
            p.grad /= train_accumulation_steps
    # momentum warmup for Muon
    frac = min(step/300, 1)
    for group in optimizer3.param_groups:
        group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # --------------- TRAINING SECTION END -------------------
    # everything that follows now is just diagnostics, prints, logging, etc.
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")

print0(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")

# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()

====================================================================================================
Running python 3.10.12 (main, Sep 11 2024, 15:47:36) [GCC 11.4.0]
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Wed Dec 11 08:39:09 2024       
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.183.06             Driver Version: 535.183.06   CUDA Version: 12.2     |
|-----------------------------------------+----------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
|                                         |                      |               MIG M. |
|=========================================+======================+======================|
|   0  NVIDIA H100 80GB HBM3          On  | 00000000:19:00.0 Off |                    0 |
| N/A   38C    P0             126W / 700W |   7084MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  | 00000000:3B:00.0 Off |                    0 |
| N/A   30C    P0             116W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  | 00000000:4C:00.0 Off |                    0 |
| N/A   29C    P0             112W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  | 00000000:5D:00.0 Off |                    0 |
| N/A   37C    P0             114W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  | 00000000:9B:00.0 Off |                    0 |
| N/A   38C    P0             119W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  | 00000000:BB:00.0 Off |                    0 |
| N/A   30C    P0             118W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  | 00000000:CB:00.0 Off |                    0 |
| N/A   36C    P0             119W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  | 00000000:DB:00.0 Off |                    0 |
| N/A   30C    P0             118W / 700W |   3211MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
                                                                                         
+---------------------------------------------------------------------------------------+
| Processes:                                                                            |
|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
|        ID   ID                                                             Usage      |
|=======================================================================================|
+---------------------------------------------------------------------------------------+

====================================================================================================
Training DataLoader: total number of tokens: 1000000000 across 10 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:28981ms step_avg:nanms
step:2/1480 train_time:29086ms step_avg:nanms
step:3/1480 train_time:29208ms step_avg:nanms
step:4/1480 train_time:29350ms step_avg:nanms
step:5/1480 train_time:29492ms step_avg:nanms
step:6/1480 train_time:29634ms step_avg:nanms
step:7/1480 train_time:29776ms step_avg:nanms
step:8/1480 train_time:29918ms step_avg:nanms
step:9/1480 train_time:30063ms step_avg:nanms
step:10/1480 train_time:30206ms step_avg:nanms
step:11/1480 train_time:143ms step_avg:nanms
step:12/1480 train_time:283ms step_avg:nanms
step:13/1480 train_time:425ms step_avg:141.67ms
step:14/1480 train_time:567ms step_avg:141.79ms
step:15/1480 train_time:708ms step_avg:141.51ms
step:16/1480 train_time:851ms step_avg:141.83ms
step:17/1480 train_time:995ms step_avg:142.16ms
step:18/1480 train_time:1139ms step_avg:142.36ms
step:19/1480 train_time:1283ms step_avg:142.52ms
step:20/1480 train_time:1424ms step_avg:142.44ms
step:21/1480 train_time:1568ms step_avg:142.51ms
step:22/1480 train_time:1710ms step_avg:142.52ms
step:23/1480 train_time:1853ms step_avg:142.54ms
step:24/1480 train_time:1997ms step_avg:142.68ms
step:25/1480 train_time:2143ms step_avg:142.83ms
step:26/1480 train_time:2284ms step_avg:142.74ms
step:27/1480 train_time:2426ms step_avg:142.68ms
step:28/1480 train_time:2568ms step_avg:142.64ms
step:29/1480 train_time:2710ms step_avg:142.65ms
step:30/1480 train_time:2853ms step_avg:142.67ms
step:31/1480 train_time:2996ms step_avg:142.67ms
step:32/1480 train_time:3139ms step_avg:142.66ms
step:33/1480 train_time:3281ms step_avg:142.65ms
step:34/1480 train_time:3424ms step_avg:142.69ms
step:35/1480 train_time:3567ms step_avg:142.69ms
step:36/1480 train_time:3709ms step_avg:142.66ms
step:37/1480 train_time:3853ms step_avg:142.69ms
step:38/1480 train_time:3997ms step_avg:142.74ms
step:39/1480 train_time:4140ms step_avg:142.77ms
step:40/1480 train_time:4283ms step_avg:142.75ms
step:41/1480 train_time:4425ms step_avg:142.75ms
step:42/1480 train_time:4567ms step_avg:142.72ms
step:43/1480 train_time:4710ms step_avg:142.73ms
step:44/1480 train_time:4854ms step_avg:142.76ms
step:45/1480 train_time:4998ms step_avg:142.79ms
step:46/1480 train_time:5142ms step_avg:142.84ms
step:47/1480 train_time:5285ms step_avg:142.83ms
step:48/1480 train_time:5427ms step_avg:142.82ms
step:49/1480 train_time:5568ms step_avg:142.77ms
step:50/1480 train_time:5711ms step_avg:142.77ms
step:51/1480 train_time:5855ms step_avg:142.80ms
step:52/1480 train_time:5997ms step_avg:142.80ms
step:53/1480 train_time:6140ms step_avg:142.79ms
step:54/1480 train_time:6281ms step_avg:142.76ms
step:55/1480 train_time:6423ms step_avg:142.74ms
step:56/1480 train_time:6565ms step_avg:142.73ms
step:57/1480 train_time:6708ms step_avg:142.73ms
step:58/1480 train_time:6852ms step_avg:142.76ms
step:59/1480 train_time:6998ms step_avg:142.82ms
step:60/1480 train_time:7141ms step_avg:142.83ms
step:61/1480 train_time:7284ms step_avg:142.81ms
step:62/1480 train_time:7425ms step_avg:142.79ms
step:63/1480 train_time:7567ms step_avg:142.77ms
step:64/1480 train_time:7709ms step_avg:142.76ms
step:65/1480 train_time:7851ms step_avg:142.74ms
step:66/1480 train_time:7994ms step_avg:142.75ms
step:67/1480 train_time:8140ms step_avg:142.81ms
step:68/1480 train_time:8284ms step_avg:142.83ms
step:69/1480 train_time:8426ms step_avg:142.82ms
step:70/1480 train_time:8569ms step_avg:142.82ms
step:71/1480 train_time:8712ms step_avg:142.81ms
step:72/1480 train_time:8857ms step_avg:142.85ms
step:73/1480 train_time:9001ms step_avg:142.87ms
step:74/1480 train_time:9143ms step_avg:142.87ms
step:75/1480 train_time:9285ms step_avg:142.85ms
step:76/1480 train_time:9429ms step_avg:142.87ms
step:77/1480 train_time:9572ms step_avg:142.86ms
step:78/1480 train_time:9714ms step_avg:142.85ms
step:79/1480 train_time:9857ms step_avg:142.86ms
step:80/1480 train_time:10372ms step_avg:148.16ms
step:81/1480 train_time:10473ms step_avg:147.50ms
step:82/1480 train_time:10614ms step_avg:147.42ms
step:83/1480 train_time:10756ms step_avg:147.34ms
step:84/1480 train_time:10899ms step_avg:147.28ms
step:85/1480 train_time:11041ms step_avg:147.22ms
step:86/1480 train_time:11184ms step_avg:147.16ms
step:87/1480 train_time:11327ms step_avg:147.10ms
step:88/1480 train_time:11470ms step_avg:147.06ms
step:89/1480 train_time:11614ms step_avg:147.01ms
step:90/1480 train_time:11759ms step_avg:146.99ms
step:91/1480 train_time:11901ms step_avg:146.93ms
step:92/1480 train_time:12044ms step_avg:146.87ms
step:93/1480 train_time:12185ms step_avg:146.81ms
step:94/1480 train_time:12327ms step_avg:146.75ms
step:95/1480 train_time:12470ms step_avg:146.71ms
step:96/1480 train_time:13003ms step_avg:151.19ms
step:97/1480 train_time:13104ms step_avg:150.62ms
step:98/1480 train_time:13624ms step_avg:154.81ms
step:99/1480 train_time:13725ms step_avg:154.22ms
step:100/1480 train_time:13867ms step_avg:154.07ms
step:101/1480 train_time:14013ms step_avg:153.99ms
step:102/1480 train_time:14151ms step_avg:153.81ms
step:103/1480 train_time:14293ms step_avg:153.69ms
step:104/1480 train_time:14437ms step_avg:153.59ms
step:105/1480 train_time:14580ms step_avg:153.47ms
step:106/1480 train_time:14724ms step_avg:153.37ms
step:107/1480 train_time:14867ms step_avg:153.27ms
step:108/1480 train_time:15009ms step_avg:153.16ms
step:109/1480 train_time:15153ms step_avg:153.06ms
step:110/1480 train_time:15297ms step_avg:152.97ms
step:111/1480 train_time:15442ms step_avg:152.89ms
step:112/1480 train_time:15587ms step_avg:152.81ms
step:113/1480 train_time:15732ms step_avg:152.74ms
step:114/1480 train_time:15878ms step_avg:152.67ms
step:115/1480 train_time:16024ms step_avg:152.61ms
step:116/1480 train_time:16169ms step_avg:152.53ms
step:117/1480 train_time:16315ms step_avg:152.48ms
step:118/1480 train_time:16462ms step_avg:152.43ms
step:119/1480 train_time:16607ms step_avg:152.36ms
step:120/1480 train_time:16752ms step_avg:152.30ms
step:121/1480 train_time:16898ms step_avg:152.24ms
step:122/1480 train_time:17044ms step_avg:152.18ms
step:123/1480 train_time:17188ms step_avg:152.11ms
step:124/1480 train_time:17334ms step_avg:152.05ms
step:125/1480 train_time:17480ms step_avg:152.00ms
step:125/1480 val_loss:4.4315 train_time:17545ms step_avg:152.57ms
step:126/1480 train_time:17636ms step_avg:152.04ms
step:127/1480 train_time:17780ms step_avg:151.96ms
step:128/1480 train_time:17927ms step_avg:151.92ms
step:129/1480 train_time:18072ms step_avg:151.87ms
step:130/1480 train_time:18216ms step_avg:151.80ms
step:131/1480 train_time:18362ms step_avg:151.75ms
step:132/1480 train_time:18509ms step_avg:151.71ms
step:133/1480 train_time:18654ms step_avg:151.66ms
step:134/1480 train_time:18800ms step_avg:151.62ms
step:135/1480 train_time:18947ms step_avg:151.57ms
step:136/1480 train_time:19093ms step_avg:151.53ms
step:137/1480 train_time:19237ms step_avg:151.47ms
step:138/1480 train_time:19384ms step_avg:151.43ms
step:139/1480 train_time:19530ms step_avg:151.40ms
step:140/1480 train_time:19676ms step_avg:151.35ms
step:141/1480 train_time:19822ms step_avg:151.31ms
step:142/1480 train_time:19968ms step_avg:151.28ms
step:143/1480 train_time:20114ms step_avg:151.23ms
step:144/1480 train_time:20258ms step_avg:151.18ms
step:145/1480 train_time:20405ms step_avg:151.14ms
step:146/1480 train_time:20550ms step_avg:151.11ms
step:147/1480 train_time:20696ms step_avg:151.07ms
step:148/1480 train_time:20843ms step_avg:151.04ms
step:149/1480 train_time:20990ms step_avg:151.01ms
step:150/1480 train_time:21135ms step_avg:150.96ms
step:151/1480 train_time:21280ms step_avg:150.93ms
step:152/1480 train_time:21427ms step_avg:150.89ms
step:153/1480 train_time:21573ms step_avg:150.86ms
step:154/1480 train_time:21717ms step_avg:150.81ms
step:155/1480 train_time:21864ms step_avg:150.78ms
step:156/1480 train_time:22012ms step_avg:150.77ms
step:157/1480 train_time:22156ms step_avg:150.72ms
step:158/1480 train_time:22302ms step_avg:150.69ms
step:159/1480 train_time:22447ms step_avg:150.65ms
step:160/1480 train_time:22592ms step_avg:150.62ms
step:161/1480 train_time:22738ms step_avg:150.58ms
step:162/1480 train_time:22885ms step_avg:150.56ms
step:163/1480 train_time:23032ms step_avg:150.53ms
step:164/1480 train_time:23176ms step_avg:150.49ms
step:165/1480 train_time:23322ms step_avg:150.46ms
step:166/1480 train_time:23469ms step_avg:150.44ms
step:167/1480 train_time:23614ms step_avg:150.41ms
step:168/1480 train_time:23759ms step_avg:150.38ms
step:169/1480 train_time:23906ms step_avg:150.35ms
step:170/1480 train_time:24051ms step_avg:150.32ms
step:171/1480 train_time:24197ms step_avg:150.29ms
step:172/1480 train_time:24343ms step_avg:150.27ms
step:173/1480 train_time:24490ms step_avg:150.24ms
step:174/1480 train_time:24634ms step_avg:150.21ms
step:175/1480 train_time:24779ms step_avg:150.18ms
step:176/1480 train_time:24925ms step_avg:150.15ms
step:177/1480 train_time:25071ms step_avg:150.13ms
step:178/1480 train_time:25215ms step_avg:150.09ms
step:179/1480 train_time:25361ms step_avg:150.07ms
step:180/1480 train_time:25891ms step_avg:152.30ms
step:181/1480 train_time:25997ms step_avg:152.03ms
step:182/1480 train_time:26144ms step_avg:152.00ms
step:183/1480 train_time:26292ms step_avg:151.97ms
step:184/1480 train_time:26435ms step_avg:151.93ms
step:185/1480 train_time:26581ms step_avg:151.89ms
step:186/1480 train_time:26727ms step_avg:151.86ms
step:187/1480 train_time:26874ms step_avg:151.83ms
step:188/1480 train_time:27020ms step_avg:151.80ms
step:189/1480 train_time:27191ms step_avg:151.90ms
step:190/1480 train_time:27313ms step_avg:151.74ms
step:191/1480 train_time:27457ms step_avg:151.70ms
step:192/1480 train_time:27604ms step_avg:151.67ms
step:193/1480 train_time:27750ms step_avg:151.64ms
step:194/1480 train_time:27896ms step_avg:151.61ms
step:195/1480 train_time:28043ms step_avg:151.58ms
step:196/1480 train_time:28190ms step_avg:151.56ms
step:197/1480 train_time:28336ms step_avg:151.53ms
step:198/1480 train_time:28483ms step_avg:151.51ms
step:199/1480 train_time:28630ms step_avg:151.48ms
step:200/1480 train_time:28775ms step_avg:151.44ms
step:201/1480 train_time:28924ms step_avg:151.43ms
step:202/1480 train_time:29067ms step_avg:151.39ms
step:203/1480 train_time:29213ms step_avg:151.36ms
step:204/1480 train_time:29357ms step_avg:151.33ms
step:205/1480 train_time:29503ms step_avg:151.30ms
step:206/1480 train_time:29650ms step_avg:151.27ms
step:207/1480 train_time:29796ms step_avg:151.25ms
step:208/1480 train_time:29942ms step_avg:151.22ms
step:209/1480 train_time:30089ms step_avg:151.20ms
step:210/1480 train_time:30235ms step_avg:151.17ms
step:211/1480 train_time:30381ms step_avg:151.15ms
step:212/1480 train_time:30529ms step_avg:151.13ms
step:213/1480 train_time:30674ms step_avg:151.10ms
step:214/1480 train_time:30821ms step_avg:151.08ms
step:215/1480 train_time:30968ms step_avg:151.06ms
step:216/1480 train_time:31113ms step_avg:151.03ms
step:217/1480 train_time:31258ms step_avg:151.00ms
step:218/1480 train_time:31404ms step_avg:150.98ms
step:219/1480 train_time:31549ms step_avg:150.95ms
step:220/1480 train_time:31695ms step_avg:150.93ms
step:221/1480 train_time:32249ms step_avg:152.84ms
step:222/1480 train_time:32757ms step_avg:154.52ms
step:223/1480 train_time:32864ms step_avg:154.29ms
step:224/1480 train_time:33013ms step_avg:154.26ms
step:225/1480 train_time:33160ms step_avg:154.23ms
step:226/1480 train_time:33309ms step_avg:154.21ms
step:227/1480 train_time:33456ms step_avg:154.18ms
step:228/1480 train_time:33606ms step_avg:154.16ms
step:229/1480 train_time:33754ms step_avg:154.13ms
step:230/1480 train_time:33903ms step_avg:154.11ms
step:231/1480 train_time:34052ms step_avg:154.08ms
step:232/1480 train_time:34201ms step_avg:154.06ms
step:233/1480 train_time:34350ms step_avg:154.03ms
step:234/1480 train_time:34497ms step_avg:154.01ms
step:235/1480 train_time:34648ms step_avg:153.99ms
step:236/1480 train_time:34795ms step_avg:153.96ms
step:237/1480 train_time:34944ms step_avg:153.94ms
step:238/1480 train_time:35093ms step_avg:153.92ms
step:239/1480 train_time:35241ms step_avg:153.89ms
step:240/1480 train_time:35390ms step_avg:153.87ms
step:241/1480 train_time:35538ms step_avg:153.84ms
step:242/1480 train_time:35688ms step_avg:153.83ms
step:243/1480 train_time:35836ms step_avg:153.80ms
step:244/1480 train_time:35986ms step_avg:153.78ms
step:245/1480 train_time:36134ms step_avg:153.76ms
step:246/1480 train_time:36283ms step_avg:153.74ms
step:247/1480 train_time:36432ms step_avg:153.72ms
step:248/1480 train_time:36580ms step_avg:153.70ms
step:249/1480 train_time:36730ms step_avg:153.68ms
step:250/1480 train_time:36878ms step_avg:153.66ms
step:250/1480 val_loss:3.9976 train_time:36945ms step_avg:153.94ms
step:251/1480 train_time:37042ms step_avg:153.70ms
step:252/1480 train_time:37185ms step_avg:153.66ms
step:253/1480 train_time:37335ms step_avg:153.64ms
step:254/1480 train_time:37482ms step_avg:153.62ms
step:255/1480 train_time:37630ms step_avg:153.59ms
step:256/1480 train_time:37778ms step_avg:153.57ms
step:257/1480 train_time:37926ms step_avg:153.55ms
step:258/1480 train_time:38076ms step_avg:153.53ms
step:259/1480 train_time:38224ms step_avg:153.51ms
step:260/1480 train_time:38373ms step_avg:153.49ms
step:261/1480 train_time:38522ms step_avg:153.47ms
step:262/1480 train_time:38669ms step_avg:153.45ms
step:263/1480 train_time:38818ms step_avg:153.43ms
step:264/1480 train_time:38966ms step_avg:153.41ms
step:265/1480 train_time:39115ms step_avg:153.39ms
step:266/1480 train_time:39262ms step_avg:153.37ms
step:267/1480 train_time:39412ms step_avg:153.35ms
step:268/1480 train_time:39561ms step_avg:153.34ms
step:269/1480 train_time:39708ms step_avg:153.31ms
step:270/1480 train_time:39857ms step_avg:153.30ms
step:271/1480 train_time:40005ms step_avg:153.27ms
step:272/1480 train_time:40153ms step_avg:153.26ms
step:273/1480 train_time:40302ms step_avg:153.24ms
step:274/1480 train_time:40449ms step_avg:153.22ms
step:275/1480 train_time:40599ms step_avg:153.20ms
step:276/1480 train_time:40746ms step_avg:153.18ms
step:277/1480 train_time:40896ms step_avg:153.17ms
step:278/1480 train_time:41043ms step_avg:153.15ms
step:279/1480 train_time:41193ms step_avg:153.13ms
step:280/1480 train_time:41341ms step_avg:153.12ms
step:281/1480 train_time:41489ms step_avg:153.10ms
step:282/1480 train_time:41639ms step_avg:153.08ms
step:283/1480 train_time:41786ms step_avg:153.06ms
step:284/1480 train_time:41937ms step_avg:153.05ms
step:285/1480 train_time:42084ms step_avg:153.03ms
step:286/1480 train_time:42233ms step_avg:153.02ms
step:287/1480 train_time:42382ms step_avg:153.00ms
step:288/1480 train_time:42529ms step_avg:152.98ms
step:289/1480 train_time:42678ms step_avg:152.97ms
step:290/1480 train_time:42827ms step_avg:152.95ms
step:291/1480 train_time:42977ms step_avg:152.94ms
step:292/1480 train_time:43124ms step_avg:152.92ms
step:293/1480 train_time:43274ms step_avg:152.91ms
step:294/1480 train_time:43422ms step_avg:152.89ms
step:295/1480 train_time:43570ms step_avg:152.88ms
step:296/1480 train_time:43719ms step_avg:152.86ms
step:297/1480 train_time:43866ms step_avg:152.84ms
step:298/1480 train_time:44016ms step_avg:152.83ms
step:299/1480 train_time:44163ms step_avg:152.81ms
step:300/1480 train_time:44312ms step_avg:152.80ms
step:301/1480 train_time:44461ms step_avg:152.79ms
step:302/1480 train_time:44609ms step_avg:152.77ms
step:303/1480 train_time:44758ms step_avg:152.76ms
step:304/1480 train_time:44906ms step_avg:152.74ms
step:305/1480 train_time:45055ms step_avg:152.73ms
step:306/1480 train_time:45203ms step_avg:152.71ms
step:307/1480 train_time:45351ms step_avg:152.70ms
step:308/1480 train_time:45501ms step_avg:152.69ms
step:309/1480 train_time:45649ms step_avg:152.67ms
step:310/1480 train_time:45798ms step_avg:152.66ms
step:311/1480 train_time:45946ms step_avg:152.64ms
step:312/1480 train_time:46094ms step_avg:152.63ms
step:313/1480 train_time:46242ms step_avg:152.61ms
step:314/1480 train_time:46391ms step_avg:152.60ms
step:315/1480 train_time:46540ms step_avg:152.59ms
step:316/1480 train_time:46688ms step_avg:152.57ms
step:317/1480 train_time:46837ms step_avg:152.56ms
step:318/1480 train_time:46985ms step_avg:152.55ms
step:319/1480 train_time:47133ms step_avg:152.53ms
step:320/1480 train_time:47281ms step_avg:152.52ms
step:321/1480 train_time:47430ms step_avg:152.51ms
step:322/1480 train_time:47579ms step_avg:152.50ms
step:323/1480 train_time:47729ms step_avg:152.49ms
step:324/1480 train_time:47878ms step_avg:152.48ms
step:325/1480 train_time:48027ms step_avg:152.47ms
step:326/1480 train_time:48176ms step_avg:152.46ms
step:327/1480 train_time:48324ms step_avg:152.44ms
step:328/1480 train_time:48473ms step_avg:152.43ms
step:329/1480 train_time:48621ms step_avg:152.42ms
step:330/1480 train_time:48771ms step_avg:152.41ms
step:331/1480 train_time:48922ms step_avg:152.41ms
step:332/1480 train_time:49073ms step_avg:152.40ms
step:333/1480 train_time:49223ms step_avg:152.39ms
step:334/1480 train_time:49374ms step_avg:152.39ms
step:335/1480 train_time:49524ms step_avg:152.38ms
step:336/1480 train_time:49676ms step_avg:152.38ms
step:337/1480 train_time:49826ms step_avg:152.37ms
step:338/1480 train_time:49977ms step_avg:152.37ms
step:339/1480 train_time:50128ms step_avg:152.36ms
step:340/1480 train_time:50279ms step_avg:152.36ms
step:341/1480 train_time:50429ms step_avg:152.35ms
step:342/1480 train_time:50581ms step_avg:152.35ms
step:343/1480 train_time:50731ms step_avg:152.35ms
step:344/1480 train_time:50882ms step_avg:152.34ms
step:345/1480 train_time:51034ms step_avg:152.34ms
step:346/1480 train_time:51185ms step_avg:152.34ms
step:347/1480 train_time:51337ms step_avg:152.33ms
step:348/1480 train_time:51487ms step_avg:152.33ms
step:349/1480 train_time:51638ms step_avg:152.33ms
step:350/1480 train_time:51789ms step_avg:152.32ms
step:351/1480 train_time:51941ms step_avg:152.32ms
step:352/1480 train_time:52091ms step_avg:152.31ms
step:353/1480 train_time:52242ms step_avg:152.31ms
step:354/1480 train_time:52393ms step_avg:152.30ms
step:355/1480 train_time:52544ms step_avg:152.30ms
step:356/1480 train_time:52695ms step_avg:152.30ms
step:357/1480 train_time:52845ms step_avg:152.29ms
step:358/1480 train_time:52997ms step_avg:152.29ms
step:359/1480 train_time:53147ms step_avg:152.28ms
step:360/1480 train_time:53299ms step_avg:152.28ms
step:361/1480 train_time:53450ms step_avg:152.28ms
step:362/1480 train_time:53601ms step_avg:152.28ms
step:363/1480 train_time:53751ms step_avg:152.27ms
step:364/1480 train_time:53902ms step_avg:152.27ms
step:365/1480 train_time:54053ms step_avg:152.26ms
step:366/1480 train_time:54204ms step_avg:152.26ms
step:367/1480 train_time:54355ms step_avg:152.25ms
step:368/1480 train_time:54504ms step_avg:152.25ms
step:369/1480 train_time:54657ms step_avg:152.25ms
step:370/1480 train_time:54806ms step_avg:152.24ms
step:371/1480 train_time:54958ms step_avg:152.24ms
step:372/1480 train_time:55107ms step_avg:152.23ms
step:373/1480 train_time:55260ms step_avg:152.23ms
step:374/1480 train_time:55409ms step_avg:152.22ms
step:375/1480 train_time:55561ms step_avg:152.22ms
step:375/1480 val_loss:3.8083 train_time:55627ms step_avg:152.40ms
step:376/1480 train_time:55718ms step_avg:152.24ms
step:377/1480 train_time:55871ms step_avg:152.24ms
step:378/1480 train_time:56022ms step_avg:152.23ms
step:379/1480 train_time:56194ms step_avg:152.29ms
step:380/1480 train_time:56323ms step_avg:152.22ms
step:381/1480 train_time:56474ms step_avg:152.22ms
step:382/1480 train_time:56624ms step_avg:152.21ms
step:383/1480 train_time:56775ms step_avg:152.21ms
step:384/1480 train_time:56926ms step_avg:152.21ms
step:385/1480 train_time:57078ms step_avg:152.21ms
step:386/1480 train_time:57229ms step_avg:152.20ms
step:387/1480 train_time:57380ms step_avg:152.20ms
step:388/1480 train_time:57530ms step_avg:152.20ms
step:389/1480 train_time:57681ms step_avg:152.19ms
step:390/1480 train_time:57833ms step_avg:152.19ms
step:391/1480 train_time:57984ms step_avg:152.19ms
step:392/1480 train_time:58135ms step_avg:152.19ms
step:393/1480 train_time:58285ms step_avg:152.18ms
step:394/1480 train_time:58436ms step_avg:152.18ms
step:395/1480 train_time:58585ms step_avg:152.17ms
step:396/1480 train_time:58737ms step_avg:152.17ms
step:397/1480 train_time:58886ms step_avg:152.16ms
step:398/1480 train_time:59037ms step_avg:152.16ms
step:399/1480 train_time:59188ms step_avg:152.15ms
step:400/1480 train_time:59339ms step_avg:152.15ms
step:401/1480 train_time:59489ms step_avg:152.15ms
step:402/1480 train_time:59640ms step_avg:152.14ms
step:403/1480 train_time:59792ms step_avg:152.14ms
step:404/1480 train_time:59943ms step_avg:152.14ms
step:405/1480 train_time:60095ms step_avg:152.14ms
step:406/1480 train_time:60245ms step_avg:152.13ms
step:407/1480 train_time:60396ms step_avg:152.13ms
step:408/1480 train_time:60545ms step_avg:152.12ms
step:409/1480 train_time:60696ms step_avg:152.12ms
step:410/1480 train_time:60846ms step_avg:152.12ms
step:411/1480 train_time:60997ms step_avg:152.11ms
step:412/1480 train_time:61148ms step_avg:152.11ms
step:413/1480 train_time:61300ms step_avg:152.11ms
step:414/1480 train_time:61451ms step_avg:152.11ms
step:415/1480 train_time:61602ms step_avg:152.10ms
step:416/1480 train_time:61754ms step_avg:152.10ms
step:417/1480 train_time:61904ms step_avg:152.10ms
step:418/1480 train_time:62056ms step_avg:152.10ms
step:419/1480 train_time:62205ms step_avg:152.09ms
step:420/1480 train_time:62357ms step_avg:152.09ms
step:421/1480 train_time:62507ms step_avg:152.09ms
step:422/1480 train_time:62659ms step_avg:152.09ms
step:423/1480 train_time:62809ms step_avg:152.08ms
step:424/1480 train_time:62961ms step_avg:152.08ms
step:425/1480 train_time:63112ms step_avg:152.08ms
step:426/1480 train_time:63262ms step_avg:152.07ms
step:427/1480 train_time:63414ms step_avg:152.07ms
step:428/1480 train_time:63564ms step_avg:152.07ms
step:429/1480 train_time:63715ms step_avg:152.06ms
step:430/1480 train_time:63865ms step_avg:152.06ms
step:431/1480 train_time:64017ms step_avg:152.06ms
step:432/1480 train_time:64166ms step_avg:152.05ms
step:433/1480 train_time:64317ms step_avg:152.05ms
step:434/1480 train_time:64467ms step_avg:152.04ms
step:435/1480 train_time:64618ms step_avg:152.04ms
step:436/1480 train_time:64767ms step_avg:152.04ms
step:437/1480 train_time:64919ms step_avg:152.04ms
step:438/1480 train_time:65069ms step_avg:152.03ms
step:439/1480 train_time:65221ms step_avg:152.03ms
step:440/1480 train_time:65373ms step_avg:152.03ms
step:441/1480 train_time:65526ms step_avg:152.03ms
step:442/1480 train_time:65680ms step_avg:152.04ms
step:443/1480 train_time:65832ms step_avg:152.04ms
step:444/1480 train_time:65985ms step_avg:152.04ms
step:445/1480 train_time:66139ms step_avg:152.04ms
step:446/1480 train_time:66292ms step_avg:152.05ms
step:447/1480 train_time:66445ms step_avg:152.05ms
step:448/1480 train_time:66598ms step_avg:152.05ms
step:449/1480 train_time:66752ms step_avg:152.05ms
step:450/1480 train_time:66906ms step_avg:152.06ms
step:451/1480 train_time:67060ms step_avg:152.06ms
step:452/1480 train_time:67213ms step_avg:152.06ms
step:453/1480 train_time:67365ms step_avg:152.07ms
step:454/1480 train_time:67518ms step_avg:152.07ms
step:455/1480 train_time:67670ms step_avg:152.07ms
step:456/1480 train_time:67822ms step_avg:152.07ms
step:457/1480 train_time:67975ms step_avg:152.07ms
step:458/1480 train_time:68128ms step_avg:152.07ms
step:459/1480 train_time:68281ms step_avg:152.07ms
step:460/1480 train_time:68434ms step_avg:152.08ms
step:461/1480 train_time:68587ms step_avg:152.08ms
step:462/1480 train_time:68740ms step_avg:152.08ms
step:463/1480 train_time:68893ms step_avg:152.08ms
step:464/1480 train_time:69045ms step_avg:152.08ms
step:465/1480 train_time:69198ms step_avg:152.08ms
step:466/1480 train_time:69352ms step_avg:152.09ms
step:467/1480 train_time:69505ms step_avg:152.09ms
step:468/1480 train_time:69659ms step_avg:152.09ms
step:469/1480 train_time:69811ms step_avg:152.09ms
step:470/1480 train_time:69965ms step_avg:152.10ms
step:471/1480 train_time:70118ms step_avg:152.10ms
step:472/1480 train_time:70270ms step_avg:152.10ms
step:473/1480 train_time:70422ms step_avg:152.10ms
step:474/1480 train_time:70576ms step_avg:152.10ms
step:475/1480 train_time:70727ms step_avg:152.10ms
step:476/1480 train_time:70882ms step_avg:152.11ms
step:477/1480 train_time:71036ms step_avg:152.11ms
step:478/1480 train_time:71189ms step_avg:152.11ms
step:479/1480 train_time:71341ms step_avg:152.11ms
step:480/1480 train_time:71494ms step_avg:152.11ms
step:481/1480 train_time:71646ms step_avg:152.11ms
step:482/1480 train_time:71799ms step_avg:152.12ms
step:483/1480 train_time:71952ms step_avg:152.12ms
step:484/1480 train_time:72106ms step_avg:152.12ms
step:485/1480 train_time:72260ms step_avg:152.13ms
step:486/1480 train_time:72413ms step_avg:152.13ms
step:487/1480 train_time:72565ms step_avg:152.13ms
step:488/1480 train_time:72719ms step_avg:152.13ms
step:489/1480 train_time:72870ms step_avg:152.13ms
step:490/1480 train_time:73023ms step_avg:152.13ms
step:491/1480 train_time:73176ms step_avg:152.13ms
step:492/1480 train_time:73329ms step_avg:152.13ms
step:493/1480 train_time:73482ms step_avg:152.14ms
step:494/1480 train_time:73636ms step_avg:152.14ms
step:495/1480 train_time:73789ms step_avg:152.14ms
step:496/1480 train_time:73942ms step_avg:152.14ms
step:497/1480 train_time:74094ms step_avg:152.14ms
step:498/1480 train_time:74246ms step_avg:152.14ms
step:499/1480 train_time:74399ms step_avg:152.15ms
step:500/1480 train_time:74552ms step_avg:152.15ms
step:500/1480 val_loss:3.6877 train_time:74622ms step_avg:152.29ms
step:501/1480 train_time:74714ms step_avg:152.17ms
step:502/1480 train_time:74866ms step_avg:152.17ms
step:503/1480 train_time:75019ms step_avg:152.17ms
step:504/1480 train_time:75172ms step_avg:152.17ms
step:505/1480 train_time:75324ms step_avg:152.17ms
step:506/1480 train_time:75478ms step_avg:152.17ms
step:507/1480 train_time:75631ms step_avg:152.17ms
step:508/1480 train_time:75783ms step_avg:152.18ms
step:509/1480 train_time:75937ms step_avg:152.18ms
step:510/1480 train_time:76089ms step_avg:152.18ms
step:511/1480 train_time:76241ms step_avg:152.18ms
step:512/1480 train_time:76394ms step_avg:152.18ms
step:513/1480 train_time:76547ms step_avg:152.18ms
step:514/1480 train_time:76701ms step_avg:152.18ms
step:515/1480 train_time:76855ms step_avg:152.19ms
step:516/1480 train_time:77008ms step_avg:152.19ms
step:517/1480 train_time:77162ms step_avg:152.19ms
step:518/1480 train_time:77315ms step_avg:152.20ms
step:519/1480 train_time:77468ms step_avg:152.20ms
step:520/1480 train_time:77621ms step_avg:152.20ms
step:521/1480 train_time:77774ms step_avg:152.20ms
step:522/1480 train_time:77928ms step_avg:152.20ms
step:523/1480 train_time:78083ms step_avg:152.21ms
step:524/1480 train_time:78236ms step_avg:152.21ms
step:525/1480 train_time:78388ms step_avg:152.21ms
step:526/1480 train_time:78540ms step_avg:152.21ms
step:527/1480 train_time:78693ms step_avg:152.21ms
step:528/1480 train_time:78848ms step_avg:152.22ms
step:529/1480 train_time:79001ms step_avg:152.22ms
step:530/1480 train_time:79155ms step_avg:152.22ms
step:531/1480 train_time:79307ms step_avg:152.22ms
step:532/1480 train_time:79459ms step_avg:152.22ms
step:533/1480 train_time:79614ms step_avg:152.23ms
step:534/1480 train_time:79766ms step_avg:152.23ms
step:535/1480 train_time:79920ms step_avg:152.23ms
step:536/1480 train_time:80074ms step_avg:152.23ms
step:537/1480 train_time:80227ms step_avg:152.23ms
step:538/1480 train_time:80381ms step_avg:152.24ms
step:539/1480 train_time:80535ms step_avg:152.24ms
step:540/1480 train_time:80688ms step_avg:152.24ms
step:541/1480 train_time:80841ms step_avg:152.24ms
step:542/1480 train_time:80993ms step_avg:152.24ms
step:543/1480 train_time:81147ms step_avg:152.25ms
step:544/1480 train_time:81300ms step_avg:152.25ms
step:545/1480 train_time:81453ms step_avg:152.25ms
step:546/1480 train_time:81606ms step_avg:152.25ms
step:547/1480 train_time:81758ms step_avg:152.25ms
step:548/1480 train_time:81913ms step_avg:152.25ms
step:549/1480 train_time:82064ms step_avg:152.25ms
step:550/1480 train_time:82218ms step_avg:152.26ms
step:551/1480 train_time:82373ms step_avg:152.26ms
step:552/1480 train_time:82528ms step_avg:152.27ms
step:553/1480 train_time:82683ms step_avg:152.27ms
step:554/1480 train_time:82837ms step_avg:152.27ms
step:555/1480 train_time:82992ms step_avg:152.28ms
step:556/1480 train_time:83146ms step_avg:152.28ms
step:557/1480 train_time:83300ms step_avg:152.29ms
step:558/1480 train_time:83455ms step_avg:152.29ms
step:559/1480 train_time:83609ms step_avg:152.29ms
step:560/1480 train_time:83763ms step_avg:152.30ms
step:561/1480 train_time:83918ms step_avg:152.30ms
step:562/1480 train_time:84073ms step_avg:152.31ms
step:563/1480 train_time:84227ms step_avg:152.31ms
step:564/1480 train_time:84381ms step_avg:152.31ms
step:565/1480 train_time:84536ms step_avg:152.32ms
step:566/1480 train_time:84690ms step_avg:152.32ms
step:567/1480 train_time:84843ms step_avg:152.32ms
step:568/1480 train_time:84997ms step_avg:152.32ms
step:569/1480 train_time:85168ms step_avg:152.36ms
step:570/1480 train_time:85307ms step_avg:152.33ms
step:571/1480 train_time:85462ms step_avg:152.34ms
step:572/1480 train_time:85617ms step_avg:152.34ms
step:573/1480 train_time:85772ms step_avg:152.35ms
step:574/1480 train_time:85929ms step_avg:152.36ms
step:575/1480 train_time:86083ms step_avg:152.36ms
step:576/1480 train_time:86238ms step_avg:152.36ms
step:577/1480 train_time:86392ms step_avg:152.37ms
step:578/1480 train_time:86549ms step_avg:152.37ms
step:579/1480 train_time:86704ms step_avg:152.38ms
step:580/1480 train_time:86858ms step_avg:152.38ms
step:581/1480 train_time:87013ms step_avg:152.39ms
step:582/1480 train_time:87167ms step_avg:152.39ms
step:583/1480 train_time:87323ms step_avg:152.40ms
step:584/1480 train_time:87477ms step_avg:152.40ms
step:585/1480 train_time:87633ms step_avg:152.41ms
step:586/1480 train_time:87787ms step_avg:152.41ms
step:587/1480 train_time:87942ms step_avg:152.41ms
step:588/1480 train_time:88096ms step_avg:152.42ms
step:589/1480 train_time:88253ms step_avg:152.42ms
step:590/1480 train_time:88407ms step_avg:152.43ms
step:591/1480 train_time:88562ms step_avg:152.43ms
step:592/1480 train_time:88716ms step_avg:152.43ms
step:593/1480 train_time:88873ms step_avg:152.44ms
step:594/1480 train_time:89028ms step_avg:152.45ms
step:595/1480 train_time:89185ms step_avg:152.45ms
step:596/1480 train_time:89340ms step_avg:152.46ms
step:597/1480 train_time:89495ms step_avg:152.46ms
step:598/1480 train_time:89650ms step_avg:152.47ms
step:599/1480 train_time:89805ms step_avg:152.47ms
step:600/1480 train_time:89961ms step_avg:152.48ms
step:601/1480 train_time:90116ms step_avg:152.48ms
step:602/1480 train_time:90272ms step_avg:152.49ms
step:603/1480 train_time:90426ms step_avg:152.49ms
step:604/1480 train_time:90581ms step_avg:152.49ms
step:605/1480 train_time:90736ms step_avg:152.50ms
step:606/1480 train_time:90891ms step_avg:152.50ms
step:607/1480 train_time:91047ms step_avg:152.51ms
step:608/1480 train_time:91203ms step_avg:152.51ms
step:609/1480 train_time:91357ms step_avg:152.52ms
step:610/1480 train_time:91513ms step_avg:152.52ms
step:611/1480 train_time:91666ms step_avg:152.52ms
step:612/1480 train_time:91821ms step_avg:152.53ms
step:613/1480 train_time:91976ms step_avg:152.53ms
step:614/1480 train_time:92132ms step_avg:152.54ms
step:615/1480 train_time:92286ms step_avg:152.54ms
step:616/1480 train_time:92440ms step_avg:152.54ms
step:617/1480 train_time:92594ms step_avg:152.54ms
step:618/1480 train_time:92749ms step_avg:152.55ms
step:619/1480 train_time:92904ms step_avg:152.55ms
step:620/1480 train_time:93059ms step_avg:152.56ms
step:621/1480 train_time:93215ms step_avg:152.56ms
step:622/1480 train_time:93369ms step_avg:152.56ms
step:623/1480 train_time:93525ms step_avg:152.57ms
step:624/1480 train_time:93681ms step_avg:152.57ms
step:625/1480 train_time:93836ms step_avg:152.58ms
step:625/1480 val_loss:3.6073 train_time:93905ms step_avg:152.69ms
step:626/1480 train_time:93997ms step_avg:152.59ms
step:627/1480 train_time:94150ms step_avg:152.59ms
step:628/1480 train_time:94304ms step_avg:152.60ms
step:629/1480 train_time:94458ms step_avg:152.60ms
step:630/1480 train_time:94611ms step_avg:152.60ms
step:631/1480 train_time:94767ms step_avg:152.60ms
step:632/1480 train_time:94920ms step_avg:152.60ms
step:633/1480 train_time:95074ms step_avg:152.61ms
step:634/1480 train_time:95229ms step_avg:152.61ms
step:635/1480 train_time:95384ms step_avg:152.61ms
step:636/1480 train_time:95539ms step_avg:152.62ms
step:637/1480 train_time:95693ms step_avg:152.62ms
step:638/1480 train_time:95849ms step_avg:152.63ms
step:639/1480 train_time:96002ms step_avg:152.63ms
step:640/1480 train_time:96155ms step_avg:152.63ms
step:641/1480 train_time:96310ms step_avg:152.63ms
step:642/1480 train_time:96465ms step_avg:152.63ms
step:643/1480 train_time:96619ms step_avg:152.64ms
step:644/1480 train_time:96773ms step_avg:152.64ms
step:645/1480 train_time:96929ms step_avg:152.64ms
step:646/1480 train_time:97083ms step_avg:152.65ms
step:647/1480 train_time:97238ms step_avg:152.65ms
step:648/1480 train_time:97394ms step_avg:152.65ms
step:649/1480 train_time:97549ms step_avg:152.66ms
step:650/1480 train_time:97704ms step_avg:152.66ms
step:651/1480 train_time:97859ms step_avg:152.67ms
step:652/1480 train_time:98013ms step_avg:152.67ms
step:653/1480 train_time:98169ms step_avg:152.67ms
step:654/1480 train_time:98324ms step_avg:152.68ms
step:655/1480 train_time:98480ms step_avg:152.68ms
step:656/1480 train_time:98634ms step_avg:152.68ms
step:657/1480 train_time:98788ms step_avg:152.69ms
step:658/1480 train_time:98944ms step_avg:152.69ms
step:659/1480 train_time:99099ms step_avg:152.69ms
step:660/1480 train_time:99255ms step_avg:152.70ms
step:661/1480 train_time:99412ms step_avg:152.71ms
step:662/1480 train_time:99568ms step_avg:152.71ms
step:663/1480 train_time:99725ms step_avg:152.72ms
step:664/1480 train_time:99881ms step_avg:152.72ms
step:665/1480 train_time:100038ms step_avg:152.73ms
step:666/1480 train_time:100195ms step_avg:152.74ms
step:667/1480 train_time:100352ms step_avg:152.74ms
step:668/1480 train_time:100509ms step_avg:152.75ms
step:669/1480 train_time:100666ms step_avg:152.76ms
step:670/1480 train_time:100822ms step_avg:152.76ms
step:671/1480 train_time:100979ms step_avg:152.77ms
step:672/1480 train_time:101134ms step_avg:152.77ms
step:673/1480 train_time:101291ms step_avg:152.78ms
step:674/1480 train_time:101447ms step_avg:152.78ms
step:675/1480 train_time:101605ms step_avg:152.79ms
step:676/1480 train_time:101763ms step_avg:152.80ms
step:677/1480 train_time:101919ms step_avg:152.80ms
step:678/1480 train_time:102074ms step_avg:152.81ms
step:679/1480 train_time:102231ms step_avg:152.81ms
step:680/1480 train_time:102387ms step_avg:152.82ms
step:681/1480 train_time:102546ms step_avg:152.83ms
step:682/1480 train_time:102705ms step_avg:152.83ms
step:683/1480 train_time:102862ms step_avg:152.84ms
step:684/1480 train_time:103018ms step_avg:152.85ms
step:685/1480 train_time:103174ms step_avg:152.85ms
step:686/1480 train_time:103331ms step_avg:152.86ms
step:687/1480 train_time:103486ms step_avg:152.86ms
step:688/1480 train_time:103644ms step_avg:152.87ms
step:689/1480 train_time:103804ms step_avg:152.88ms
step:690/1480 train_time:103960ms step_avg:152.88ms
step:691/1480 train_time:104117ms step_avg:152.89ms
step:692/1480 train_time:104274ms step_avg:152.89ms
step:693/1480 train_time:104431ms step_avg:152.90ms
step:694/1480 train_time:104587ms step_avg:152.90ms
step:695/1480 train_time:104743ms step_avg:152.91ms
step:696/1480 train_time:104898ms step_avg:152.91ms
step:697/1480 train_time:105055ms step_avg:152.92ms
step:698/1480 train_time:105210ms step_avg:152.92ms
step:699/1480 train_time:105368ms step_avg:152.93ms
step:700/1480 train_time:105525ms step_avg:152.94ms
step:701/1480 train_time:105681ms step_avg:152.94ms
step:702/1480 train_time:105836ms step_avg:152.94ms
step:703/1480 train_time:105991ms step_avg:152.95ms
step:704/1480 train_time:106147ms step_avg:152.95ms
step:705/1480 train_time:106303ms step_avg:152.95ms
step:706/1480 train_time:106459ms step_avg:152.96ms
step:707/1480 train_time:106615ms step_avg:152.96ms
step:708/1480 train_time:106772ms step_avg:152.97ms
step:709/1480 train_time:106928ms step_avg:152.97ms
step:710/1480 train_time:107083ms step_avg:152.98ms
step:711/1480 train_time:107239ms step_avg:152.98ms
step:712/1480 train_time:107397ms step_avg:152.99ms
step:713/1480 train_time:107553ms step_avg:152.99ms
step:714/1480 train_time:107710ms step_avg:153.00ms
step:715/1480 train_time:107866ms step_avg:153.00ms
step:716/1480 train_time:108021ms step_avg:153.00ms
step:717/1480 train_time:108178ms step_avg:153.01ms
step:718/1480 train_time:108333ms step_avg:153.01ms
step:719/1480 train_time:108489ms step_avg:153.02ms
step:720/1480 train_time:108647ms step_avg:153.02ms
step:721/1480 train_time:108804ms step_avg:153.03ms
step:722/1480 train_time:108961ms step_avg:153.04ms
step:723/1480 train_time:109118ms step_avg:153.04ms
step:724/1480 train_time:109273ms step_avg:153.04ms
step:725/1480 train_time:109431ms step_avg:153.05ms
step:726/1480 train_time:109587ms step_avg:153.05ms
step:727/1480 train_time:109745ms step_avg:153.06ms
step:728/1480 train_time:109902ms step_avg:153.07ms
step:729/1480 train_time:110057ms step_avg:153.07ms
step:730/1480 train_time:110215ms step_avg:153.08ms
step:731/1480 train_time:110371ms step_avg:153.08ms
step:732/1480 train_time:110528ms step_avg:153.09ms
step:733/1480 train_time:110684ms step_avg:153.09ms
step:734/1480 train_time:110841ms step_avg:153.10ms
step:735/1480 train_time:110998ms step_avg:153.10ms
step:736/1480 train_time:111154ms step_avg:153.10ms
step:737/1480 train_time:111310ms step_avg:153.11ms
step:738/1480 train_time:111466ms step_avg:153.11ms
step:739/1480 train_time:111622ms step_avg:153.12ms
step:740/1480 train_time:111781ms step_avg:153.13ms
step:741/1480 train_time:111940ms step_avg:153.13ms
step:742/1480 train_time:112095ms step_avg:153.14ms
step:743/1480 train_time:112251ms step_avg:153.14ms
step:744/1480 train_time:112407ms step_avg:153.14ms
step:745/1480 train_time:112564ms step_avg:153.15ms
step:746/1480 train_time:112721ms step_avg:153.15ms
step:747/1480 train_time:112879ms step_avg:153.16ms
step:748/1480 train_time:113038ms step_avg:153.17ms
step:749/1480 train_time:113195ms step_avg:153.17ms
step:750/1480 train_time:113351ms step_avg:153.18ms
step:750/1480 val_loss:3.5515 train_time:113422ms step_avg:153.27ms
step:751/1480 train_time:113516ms step_avg:153.19ms
step:752/1480 train_time:113671ms step_avg:153.20ms
step:753/1480 train_time:113827ms step_avg:153.20ms
step:754/1480 train_time:113982ms step_avg:153.20ms
step:755/1480 train_time:114140ms step_avg:153.21ms
step:756/1480 train_time:114296ms step_avg:153.21ms
step:757/1480 train_time:114454ms step_avg:153.22ms
step:758/1480 train_time:114610ms step_avg:153.22ms
step:759/1480 train_time:114784ms step_avg:153.25ms
step:760/1480 train_time:114922ms step_avg:153.23ms
step:761/1480 train_time:115079ms step_avg:153.23ms
step:762/1480 train_time:115234ms step_avg:153.24ms
step:763/1480 train_time:115392ms step_avg:153.24ms
step:764/1480 train_time:115550ms step_avg:153.25ms
step:765/1480 train_time:115707ms step_avg:153.25ms
step:766/1480 train_time:115865ms step_avg:153.26ms
step:767/1480 train_time:116023ms step_avg:153.27ms
step:768/1480 train_time:116180ms step_avg:153.27ms
step:769/1480 train_time:116337ms step_avg:153.28ms
step:770/1480 train_time:116494ms step_avg:153.28ms
step:771/1480 train_time:116650ms step_avg:153.29ms
step:772/1480 train_time:116808ms step_avg:153.29ms
step:773/1480 train_time:116965ms step_avg:153.30ms
step:774/1480 train_time:117123ms step_avg:153.30ms
step:775/1480 train_time:117281ms step_avg:153.31ms
step:776/1480 train_time:117439ms step_avg:153.31ms
step:777/1480 train_time:117598ms step_avg:153.32ms
step:778/1480 train_time:117756ms step_avg:153.33ms
step:779/1480 train_time:117914ms step_avg:153.33ms
step:780/1480 train_time:118073ms step_avg:153.34ms
step:781/1480 train_time:118230ms step_avg:153.35ms
step:782/1480 train_time:118389ms step_avg:153.35ms
step:783/1480 train_time:118547ms step_avg:153.36ms
step:784/1480 train_time:118704ms step_avg:153.36ms
step:785/1480 train_time:118861ms step_avg:153.37ms
step:786/1480 train_time:119020ms step_avg:153.38ms
step:787/1480 train_time:119177ms step_avg:153.38ms
step:788/1480 train_time:119335ms step_avg:153.39ms
step:789/1480 train_time:119491ms step_avg:153.39ms
step:790/1480 train_time:119650ms step_avg:153.40ms
step:791/1480 train_time:119809ms step_avg:153.40ms
step:792/1480 train_time:119967ms step_avg:153.41ms
step:793/1480 train_time:120125ms step_avg:153.42ms
step:794/1480 train_time:120284ms step_avg:153.42ms
step:795/1480 train_time:120445ms step_avg:153.43ms
step:796/1480 train_time:120603ms step_avg:153.44ms
step:797/1480 train_time:120760ms step_avg:153.44ms
step:798/1480 train_time:120919ms step_avg:153.45ms
step:799/1480 train_time:121079ms step_avg:153.46ms
step:800/1480 train_time:121236ms step_avg:153.46ms
step:801/1480 train_time:121394ms step_avg:153.47ms
step:802/1480 train_time:121552ms step_avg:153.47ms
step:803/1480 train_time:121710ms step_avg:153.48ms
step:804/1480 train_time:121867ms step_avg:153.49ms
step:805/1480 train_time:122026ms step_avg:153.49ms
step:806/1480 train_time:122183ms step_avg:153.50ms
step:807/1480 train_time:122340ms step_avg:153.50ms
step:808/1480 train_time:122498ms step_avg:153.51ms
step:809/1480 train_time:122654ms step_avg:153.51ms
step:810/1480 train_time:122812ms step_avg:153.51ms
step:811/1480 train_time:122971ms step_avg:153.52ms
step:812/1480 train_time:123128ms step_avg:153.53ms
step:813/1480 train_time:123285ms step_avg:153.53ms
step:814/1480 train_time:123442ms step_avg:153.54ms
step:815/1480 train_time:123600ms step_avg:153.54ms
step:816/1480 train_time:123760ms step_avg:153.55ms
step:817/1480 train_time:123919ms step_avg:153.55ms
step:818/1480 train_time:124076ms step_avg:153.56ms
step:819/1480 train_time:124233ms step_avg:153.56ms
step:820/1480 train_time:124390ms step_avg:153.57ms
step:821/1480 train_time:124546ms step_avg:153.57ms
step:822/1480 train_time:124704ms step_avg:153.58ms
step:823/1480 train_time:124862ms step_avg:153.58ms
step:824/1480 train_time:125019ms step_avg:153.59ms
step:825/1480 train_time:125179ms step_avg:153.59ms
step:826/1480 train_time:125338ms step_avg:153.60ms
step:827/1480 train_time:125497ms step_avg:153.61ms
step:828/1480 train_time:125655ms step_avg:153.61ms
step:829/1480 train_time:125813ms step_avg:153.62ms
step:830/1480 train_time:125974ms step_avg:153.63ms
step:831/1480 train_time:126131ms step_avg:153.63ms
step:832/1480 train_time:126290ms step_avg:153.64ms
step:833/1480 train_time:126448ms step_avg:153.64ms
step:834/1480 train_time:126609ms step_avg:153.65ms
step:835/1480 train_time:126766ms step_avg:153.66ms
step:836/1480 train_time:126924ms step_avg:153.66ms
step:837/1480 train_time:127081ms step_avg:153.67ms
step:838/1480 train_time:127239ms step_avg:153.67ms
step:839/1480 train_time:127397ms step_avg:153.68ms
step:840/1480 train_time:127554ms step_avg:153.68ms
step:841/1480 train_time:127711ms step_avg:153.68ms
step:842/1480 train_time:127869ms step_avg:153.69ms
step:843/1480 train_time:128027ms step_avg:153.69ms
step:844/1480 train_time:128183ms step_avg:153.70ms
step:845/1480 train_time:128340ms step_avg:153.70ms
step:846/1480 train_time:128500ms step_avg:153.71ms
step:847/1480 train_time:128659ms step_avg:153.71ms
step:848/1480 train_time:128818ms step_avg:153.72ms
step:849/1480 train_time:128976ms step_avg:153.73ms
step:850/1480 train_time:129134ms step_avg:153.73ms
step:851/1480 train_time:129293ms step_avg:153.74ms
step:852/1480 train_time:129451ms step_avg:153.74ms
step:853/1480 train_time:129609ms step_avg:153.75ms
step:854/1480 train_time:129767ms step_avg:153.75ms
step:855/1480 train_time:129924ms step_avg:153.76ms
step:856/1480 train_time:130082ms step_avg:153.76ms
step:857/1480 train_time:130240ms step_avg:153.77ms
step:858/1480 train_time:130401ms step_avg:153.77ms
step:859/1480 train_time:130559ms step_avg:153.78ms
step:860/1480 train_time:130717ms step_avg:153.78ms
step:861/1480 train_time:130876ms step_avg:153.79ms
step:862/1480 train_time:131037ms step_avg:153.80ms
step:863/1480 train_time:131196ms step_avg:153.81ms
step:864/1480 train_time:131355ms step_avg:153.81ms
step:865/1480 train_time:131513ms step_avg:153.82ms
step:866/1480 train_time:131672ms step_avg:153.82ms
step:867/1480 train_time:131832ms step_avg:153.83ms
step:868/1480 train_time:131989ms step_avg:153.83ms
step:869/1480 train_time:132147ms step_avg:153.84ms
step:870/1480 train_time:132305ms step_avg:153.84ms
step:871/1480 train_time:132462ms step_avg:153.85ms
step:872/1480 train_time:132622ms step_avg:153.85ms
step:873/1480 train_time:132778ms step_avg:153.86ms
step:874/1480 train_time:132938ms step_avg:153.86ms
step:875/1480 train_time:133097ms step_avg:153.87ms
step:875/1480 val_loss:3.5080 train_time:133168ms step_avg:153.95ms
step:876/1480 train_time:133259ms step_avg:153.88ms
step:877/1480 train_time:133415ms step_avg:153.88ms
step:878/1480 train_time:133572ms step_avg:153.88ms
step:879/1480 train_time:133731ms step_avg:153.89ms
step:880/1480 train_time:133889ms step_avg:153.90ms
step:881/1480 train_time:134048ms step_avg:153.90ms
step:882/1480 train_time:134206ms step_avg:153.91ms
step:883/1480 train_time:134368ms step_avg:153.92ms
step:884/1480 train_time:134530ms step_avg:153.92ms
step:885/1480 train_time:134691ms step_avg:153.93ms
step:886/1480 train_time:134849ms step_avg:153.94ms
step:887/1480 train_time:135008ms step_avg:153.94ms
step:888/1480 train_time:135172ms step_avg:153.95ms
step:889/1480 train_time:135335ms step_avg:153.96ms
step:890/1480 train_time:135493ms step_avg:153.97ms
step:891/1480 train_time:135653ms step_avg:153.98ms
step:892/1480 train_time:135814ms step_avg:153.98ms
step:893/1480 train_time:135972ms step_avg:153.99ms
step:894/1480 train_time:136131ms step_avg:153.99ms
step:895/1480 train_time:136291ms step_avg:154.00ms
step:896/1480 train_time:136450ms step_avg:154.01ms
step:897/1480 train_time:136613ms step_avg:154.02ms
step:898/1480 train_time:136773ms step_avg:154.02ms
step:899/1480 train_time:136933ms step_avg:154.03ms
step:900/1480 train_time:137091ms step_avg:154.03ms
step:901/1480 train_time:137250ms step_avg:154.04ms
step:902/1480 train_time:137407ms step_avg:154.04ms
step:903/1480 train_time:137571ms step_avg:154.05ms
step:904/1480 train_time:137731ms step_avg:154.06ms
step:905/1480 train_time:137890ms step_avg:154.07ms
step:906/1480 train_time:138049ms step_avg:154.07ms
step:907/1480 train_time:138211ms step_avg:154.08ms
step:908/1480 train_time:138370ms step_avg:154.09ms
step:909/1480 train_time:138529ms step_avg:154.09ms
step:910/1480 train_time:138693ms step_avg:154.10ms
step:911/1480 train_time:138852ms step_avg:154.11ms
step:912/1480 train_time:139012ms step_avg:154.12ms
step:913/1480 train_time:139173ms step_avg:154.12ms
step:914/1480 train_time:139334ms step_avg:154.13ms
step:915/1480 train_time:139495ms step_avg:154.14ms
step:916/1480 train_time:139653ms step_avg:154.14ms
step:917/1480 train_time:139811ms step_avg:154.15ms
step:918/1480 train_time:139973ms step_avg:154.15ms
step:919/1480 train_time:140135ms step_avg:154.16ms
step:920/1480 train_time:140295ms step_avg:154.17ms
step:921/1480 train_time:140453ms step_avg:154.17ms
step:922/1480 train_time:140613ms step_avg:154.18ms
step:923/1480 train_time:140771ms step_avg:154.19ms
step:924/1480 train_time:140929ms step_avg:154.19ms
step:925/1480 train_time:141090ms step_avg:154.20ms
step:926/1480 train_time:141249ms step_avg:154.20ms
step:927/1480 train_time:141407ms step_avg:154.21ms
step:928/1480 train_time:141568ms step_avg:154.21ms
step:929/1480 train_time:141728ms step_avg:154.22ms
step:930/1480 train_time:141887ms step_avg:154.22ms
step:931/1480 train_time:142046ms step_avg:154.23ms
step:932/1480 train_time:142204ms step_avg:154.23ms
step:933/1480 train_time:142364ms step_avg:154.24ms
step:934/1480 train_time:142524ms step_avg:154.25ms
step:935/1480 train_time:142687ms step_avg:154.26ms
step:936/1480 train_time:142847ms step_avg:154.26ms
step:937/1480 train_time:143008ms step_avg:154.27ms
step:938/1480 train_time:143165ms step_avg:154.27ms
step:939/1480 train_time:143326ms step_avg:154.28ms
step:940/1480 train_time:143487ms step_avg:154.29ms
step:941/1480 train_time:143646ms step_avg:154.29ms
step:942/1480 train_time:143803ms step_avg:154.30ms
step:943/1480 train_time:143963ms step_avg:154.30ms
step:944/1480 train_time:144124ms step_avg:154.31ms
step:945/1480 train_time:144284ms step_avg:154.31ms
step:946/1480 train_time:144445ms step_avg:154.32ms
step:947/1480 train_time:144605ms step_avg:154.33ms
step:948/1480 train_time:144766ms step_avg:154.33ms
step:949/1480 train_time:144943ms step_avg:154.36ms
step:950/1480 train_time:145083ms step_avg:154.34ms
step:951/1480 train_time:145246ms step_avg:154.35ms
step:952/1480 train_time:145405ms step_avg:154.36ms
step:953/1480 train_time:145567ms step_avg:154.37ms
step:954/1480 train_time:145728ms step_avg:154.37ms
step:955/1480 train_time:145886ms step_avg:154.38ms
step:956/1480 train_time:146046ms step_avg:154.38ms
step:957/1480 train_time:146204ms step_avg:154.39ms
step:958/1480 train_time:146369ms step_avg:154.40ms
step:959/1480 train_time:146530ms step_avg:154.40ms
step:960/1480 train_time:146692ms step_avg:154.41ms
step:961/1480 train_time:146851ms step_avg:154.42ms
step:962/1480 train_time:147009ms step_avg:154.42ms
step:963/1480 train_time:147170ms step_avg:154.43ms
step:964/1480 train_time:147333ms step_avg:154.44ms
step:965/1480 train_time:147494ms step_avg:154.44ms
step:966/1480 train_time:147651ms step_avg:154.45ms
step:967/1480 train_time:147808ms step_avg:154.45ms
step:968/1480 train_time:147967ms step_avg:154.45ms
step:969/1480 train_time:148130ms step_avg:154.46ms
step:970/1480 train_time:148287ms step_avg:154.47ms
step:971/1480 train_time:148446ms step_avg:154.47ms
step:972/1480 train_time:148605ms step_avg:154.48ms
step:973/1480 train_time:148763ms step_avg:154.48ms
step:974/1480 train_time:148924ms step_avg:154.49ms
step:975/1480 train_time:149083ms step_avg:154.49ms
step:976/1480 train_time:149243ms step_avg:154.50ms
step:977/1480 train_time:149401ms step_avg:154.50ms
step:978/1480 train_time:149561ms step_avg:154.50ms
step:979/1480 train_time:149720ms step_avg:154.51ms
step:980/1480 train_time:149879ms step_avg:154.51ms
step:981/1480 train_time:150040ms step_avg:154.52ms
step:982/1480 train_time:150200ms step_avg:154.53ms
step:983/1480 train_time:150360ms step_avg:154.53ms
step:984/1480 train_time:150518ms step_avg:154.54ms
step:985/1480 train_time:150680ms step_avg:154.54ms
step:986/1480 train_time:150838ms step_avg:154.55ms
step:987/1480 train_time:150997ms step_avg:154.55ms
step:988/1480 train_time:151156ms step_avg:154.56ms
step:989/1480 train_time:151315ms step_avg:154.56ms
step:990/1480 train_time:151476ms step_avg:154.57ms
step:991/1480 train_time:151636ms step_avg:154.57ms
step:992/1480 train_time:151801ms step_avg:154.58ms
step:993/1480 train_time:151971ms step_avg:154.60ms
step:994/1480 train_time:152131ms step_avg:154.60ms
step:995/1480 train_time:152290ms step_avg:154.61ms
step:996/1480 train_time:152448ms step_avg:154.61ms
step:997/1480 train_time:152607ms step_avg:154.62ms
step:998/1480 train_time:152767ms step_avg:154.62ms
step:999/1480 train_time:152928ms step_avg:154.63ms
step:1000/1480 train_time:153089ms step_avg:154.63ms
step:1000/1480 val_loss:3.4430 train_time:153161ms step_avg:154.71ms
step:1001/1480 train_time:153257ms step_avg:154.65ms
step:1002/1480 train_time:153412ms step_avg:154.65ms
step:1003/1480 train_time:153576ms step_avg:154.66ms
step:1004/1480 train_time:153737ms step_avg:154.67ms
step:1005/1480 train_time:153896ms step_avg:154.67ms
step:1006/1480 train_time:154056ms step_avg:154.67ms
step:1007/1480 train_time:154215ms step_avg:154.68ms
step:1008/1480 train_time:154375ms step_avg:154.68ms
step:1009/1480 train_time:154540ms step_avg:154.69ms
step:1010/1480 train_time:154699ms step_avg:154.70ms
step:1011/1480 train_time:154859ms step_avg:154.70ms
step:1012/1480 train_time:155018ms step_avg:154.71ms
step:1013/1480 train_time:155179ms step_avg:154.72ms
step:1014/1480 train_time:155341ms step_avg:154.72ms
step:1015/1480 train_time:155502ms step_avg:154.73ms
step:1016/1480 train_time:155663ms step_avg:154.73ms
step:1017/1480 train_time:155825ms step_avg:154.74ms
step:1018/1480 train_time:155985ms step_avg:154.75ms
step:1019/1480 train_time:156146ms step_avg:154.75ms
step:1020/1480 train_time:156307ms step_avg:154.76ms
step:1021/1480 train_time:156466ms step_avg:154.76ms
step:1022/1480 train_time:156627ms step_avg:154.77ms
step:1023/1480 train_time:156791ms step_avg:154.78ms
step:1024/1480 train_time:156951ms step_avg:154.78ms
step:1025/1480 train_time:157113ms step_avg:154.79ms
step:1026/1480 train_time:157274ms step_avg:154.80ms
step:1027/1480 train_time:157434ms step_avg:154.80ms
step:1028/1480 train_time:157597ms step_avg:154.81ms
step:1029/1480 train_time:157759ms step_avg:154.82ms
step:1030/1480 train_time:157919ms step_avg:154.82ms
step:1031/1480 train_time:158077ms step_avg:154.83ms
step:1032/1480 train_time:158242ms step_avg:154.84ms
step:1033/1480 train_time:158400ms step_avg:154.84ms
step:1034/1480 train_time:158560ms step_avg:154.84ms
step:1035/1480 train_time:158722ms step_avg:154.85ms
step:1036/1480 train_time:158883ms step_avg:154.86ms
step:1037/1480 train_time:159044ms step_avg:154.86ms
step:1038/1480 train_time:159203ms step_avg:154.87ms
step:1039/1480 train_time:159366ms step_avg:154.87ms
step:1040/1480 train_time:159527ms step_avg:154.88ms
step:1041/1480 train_time:159689ms step_avg:154.89ms
step:1042/1480 train_time:159849ms step_avg:154.89ms
step:1043/1480 train_time:160008ms step_avg:154.90ms
step:1044/1480 train_time:160166ms step_avg:154.90ms
step:1045/1480 train_time:160327ms step_avg:154.91ms
step:1046/1480 train_time:160490ms step_avg:154.91ms
step:1047/1480 train_time:160650ms step_avg:154.92ms
step:1048/1480 train_time:160812ms step_avg:154.92ms
step:1049/1480 train_time:160973ms step_avg:154.93ms
step:1050/1480 train_time:161134ms step_avg:154.94ms
step:1051/1480 train_time:161294ms step_avg:154.94ms
step:1052/1480 train_time:161456ms step_avg:154.95ms
step:1053/1480 train_time:161618ms step_avg:154.96ms
step:1054/1480 train_time:161778ms step_avg:154.96ms
step:1055/1480 train_time:161938ms step_avg:154.96ms
step:1056/1480 train_time:162096ms step_avg:154.97ms
step:1057/1480 train_time:162258ms step_avg:154.97ms
step:1058/1480 train_time:162419ms step_avg:154.98ms
step:1059/1480 train_time:162581ms step_avg:154.99ms
step:1060/1480 train_time:162742ms step_avg:154.99ms
step:1061/1480 train_time:162901ms step_avg:155.00ms
step:1062/1480 train_time:163061ms step_avg:155.00ms
step:1063/1480 train_time:163221ms step_avg:155.01ms
step:1064/1480 train_time:163379ms step_avg:155.01ms
step:1065/1480 train_time:163539ms step_avg:155.01ms
step:1066/1480 train_time:163703ms step_avg:155.02ms
step:1067/1480 train_time:163866ms step_avg:155.03ms
step:1068/1480 train_time:164026ms step_avg:155.03ms
step:1069/1480 train_time:164191ms step_avg:155.04ms
step:1070/1480 train_time:164352ms step_avg:155.05ms
step:1071/1480 train_time:164516ms step_avg:155.06ms
step:1072/1480 train_time:164674ms step_avg:155.06ms
step:1073/1480 train_time:164833ms step_avg:155.06ms
step:1074/1480 train_time:164992ms step_avg:155.07ms
step:1075/1480 train_time:165154ms step_avg:155.07ms
step:1076/1480 train_time:165314ms step_avg:155.08ms
step:1077/1480 train_time:165473ms step_avg:155.08ms
step:1078/1480 train_time:165638ms step_avg:155.09ms
step:1079/1480 train_time:165801ms step_avg:155.10ms
step:1080/1480 train_time:165962ms step_avg:155.10ms
step:1081/1480 train_time:166121ms step_avg:155.11ms
step:1082/1480 train_time:166281ms step_avg:155.11ms
step:1083/1480 train_time:166440ms step_avg:155.12ms
step:1084/1480 train_time:166601ms step_avg:155.12ms
step:1085/1480 train_time:166760ms step_avg:155.13ms
step:1086/1480 train_time:166922ms step_avg:155.13ms
step:1087/1480 train_time:167083ms step_avg:155.14ms
step:1088/1480 train_time:167243ms step_avg:155.14ms
step:1089/1480 train_time:167407ms step_avg:155.15ms
step:1090/1480 train_time:167571ms step_avg:155.16ms
step:1091/1480 train_time:167733ms step_avg:155.16ms
step:1092/1480 train_time:167894ms step_avg:155.17ms
step:1093/1480 train_time:168056ms step_avg:155.18ms
step:1094/1480 train_time:168216ms step_avg:155.18ms
step:1095/1480 train_time:168376ms step_avg:155.19ms
step:1096/1480 train_time:168538ms step_avg:155.19ms
step:1097/1480 train_time:168699ms step_avg:155.20ms
step:1098/1480 train_time:168862ms step_avg:155.20ms
step:1099/1480 train_time:169024ms step_avg:155.21ms
step:1100/1480 train_time:169188ms step_avg:155.22ms
step:1101/1480 train_time:169351ms step_avg:155.23ms
step:1102/1480 train_time:169514ms step_avg:155.23ms
step:1103/1480 train_time:169679ms step_avg:155.24ms
step:1104/1480 train_time:169840ms step_avg:155.25ms
step:1105/1480 train_time:170001ms step_avg:155.25ms
step:1106/1480 train_time:170162ms step_avg:155.26ms
step:1107/1480 train_time:170324ms step_avg:155.26ms
step:1108/1480 train_time:170484ms step_avg:155.27ms
step:1109/1480 train_time:170644ms step_avg:155.27ms
step:1110/1480 train_time:170806ms step_avg:155.28ms
step:1111/1480 train_time:170968ms step_avg:155.28ms
step:1112/1480 train_time:171131ms step_avg:155.29ms
step:1113/1480 train_time:171300ms step_avg:155.30ms
step:1114/1480 train_time:171462ms step_avg:155.31ms
step:1115/1480 train_time:171623ms step_avg:155.32ms
step:1116/1480 train_time:171783ms step_avg:155.32ms
step:1117/1480 train_time:171947ms step_avg:155.33ms
step:1118/1480 train_time:172115ms step_avg:155.34ms
step:1119/1480 train_time:172275ms step_avg:155.34ms
step:1120/1480 train_time:172435ms step_avg:155.35ms
step:1121/1480 train_time:172596ms step_avg:155.35ms
step:1122/1480 train_time:172758ms step_avg:155.36ms
step:1123/1480 train_time:172918ms step_avg:155.36ms
step:1124/1480 train_time:173081ms step_avg:155.37ms
step:1125/1480 train_time:173242ms step_avg:155.37ms
step:1125/1480 val_loss:3.3879 train_time:173317ms step_avg:155.44ms
step:1126/1480 train_time:173408ms step_avg:155.38ms
step:1127/1480 train_time:173568ms step_avg:155.39ms
step:1128/1480 train_time:173728ms step_avg:155.39ms
step:1129/1480 train_time:173893ms step_avg:155.40ms
step:1130/1480 train_time:174053ms step_avg:155.40ms
step:1131/1480 train_time:174219ms step_avg:155.41ms
step:1132/1480 train_time:174378ms step_avg:155.42ms
step:1133/1480 train_time:174543ms step_avg:155.43ms
step:1134/1480 train_time:174706ms step_avg:155.43ms
step:1135/1480 train_time:174868ms step_avg:155.44ms
step:1136/1480 train_time:175031ms step_avg:155.44ms
step:1137/1480 train_time:175192ms step_avg:155.45ms
step:1138/1480 train_time:175355ms step_avg:155.46ms
step:1139/1480 train_time:175539ms step_avg:155.48ms
step:1140/1480 train_time:175680ms step_avg:155.47ms
step:1141/1480 train_time:175844ms step_avg:155.48ms
step:1142/1480 train_time:176004ms step_avg:155.48ms
step:1143/1480 train_time:176169ms step_avg:155.49ms
step:1144/1480 train_time:176330ms step_avg:155.49ms
step:1145/1480 train_time:176490ms step_avg:155.50ms
step:1146/1480 train_time:176652ms step_avg:155.50ms
step:1147/1480 train_time:176814ms step_avg:155.51ms
step:1148/1480 train_time:176975ms step_avg:155.51ms
step:1149/1480 train_time:177136ms step_avg:155.52ms
step:1150/1480 train_time:177296ms step_avg:155.52ms
step:1151/1480 train_time:177458ms step_avg:155.53ms
step:1152/1480 train_time:177621ms step_avg:155.54ms
step:1153/1480 train_time:177786ms step_avg:155.54ms
step:1154/1480 train_time:177948ms step_avg:155.55ms
step:1155/1480 train_time:178111ms step_avg:155.56ms
step:1156/1480 train_time:178277ms step_avg:155.56ms
step:1157/1480 train_time:178441ms step_avg:155.57ms
step:1158/1480 train_time:178601ms step_avg:155.58ms
step:1159/1480 train_time:178763ms step_avg:155.58ms
step:1160/1480 train_time:178924ms step_avg:155.59ms
step:1161/1480 train_time:179086ms step_avg:155.59ms
step:1162/1480 train_time:179250ms step_avg:155.60ms
step:1163/1480 train_time:179415ms step_avg:155.61ms
step:1164/1480 train_time:179576ms step_avg:155.61ms
step:1165/1480 train_time:179735ms step_avg:155.61ms
step:1166/1480 train_time:179896ms step_avg:155.62ms
step:1167/1480 train_time:180055ms step_avg:155.62ms
step:1168/1480 train_time:180216ms step_avg:155.63ms
step:1169/1480 train_time:180379ms step_avg:155.63ms
step:1170/1480 train_time:180541ms step_avg:155.64ms
step:1171/1480 train_time:180703ms step_avg:155.64ms
step:1172/1480 train_time:180863ms step_avg:155.65ms
step:1173/1480 train_time:181027ms step_avg:155.65ms
step:1174/1480 train_time:181198ms step_avg:155.67ms
step:1175/1480 train_time:181360ms step_avg:155.67ms
step:1176/1480 train_time:181526ms step_avg:155.68ms
step:1177/1480 train_time:181694ms step_avg:155.69ms
step:1178/1480 train_time:181853ms step_avg:155.70ms
step:1179/1480 train_time:182013ms step_avg:155.70ms
step:1180/1480 train_time:182181ms step_avg:155.71ms
step:1181/1480 train_time:182342ms step_avg:155.72ms
step:1182/1480 train_time:182503ms step_avg:155.72ms
step:1183/1480 train_time:182665ms step_avg:155.72ms
step:1184/1480 train_time:182827ms step_avg:155.73ms
step:1185/1480 train_time:182992ms step_avg:155.74ms
step:1186/1480 train_time:183154ms step_avg:155.74ms
step:1187/1480 train_time:183325ms step_avg:155.76ms
step:1188/1480 train_time:183484ms step_avg:155.76ms
step:1189/1480 train_time:183646ms step_avg:155.76ms
step:1190/1480 train_time:183806ms step_avg:155.77ms
step:1191/1480 train_time:183971ms step_avg:155.78ms
step:1192/1480 train_time:184132ms step_avg:155.78ms
step:1193/1480 train_time:184293ms step_avg:155.78ms
step:1194/1480 train_time:184452ms step_avg:155.79ms
step:1195/1480 train_time:184615ms step_avg:155.79ms
step:1196/1480 train_time:184787ms step_avg:155.81ms
step:1197/1480 train_time:184949ms step_avg:155.81ms
step:1198/1480 train_time:185117ms step_avg:155.82ms
step:1199/1480 train_time:185278ms step_avg:155.83ms
step:1200/1480 train_time:185440ms step_avg:155.83ms
step:1201/1480 train_time:185601ms step_avg:155.84ms
step:1202/1480 train_time:185770ms step_avg:155.85ms
step:1203/1480 train_time:185936ms step_avg:155.86ms
step:1204/1480 train_time:186099ms step_avg:155.86ms
step:1205/1480 train_time:186259ms step_avg:155.87ms
step:1206/1480 train_time:186420ms step_avg:155.87ms
step:1207/1480 train_time:186582ms step_avg:155.87ms
step:1208/1480 train_time:186743ms step_avg:155.88ms
step:1209/1480 train_time:186908ms step_avg:155.89ms
step:1210/1480 train_time:187073ms step_avg:155.89ms
step:1211/1480 train_time:187236ms step_avg:155.90ms
step:1212/1480 train_time:187397ms step_avg:155.90ms
step:1213/1480 train_time:187561ms step_avg:155.91ms
step:1214/1480 train_time:187727ms step_avg:155.92ms
step:1215/1480 train_time:187892ms step_avg:155.93ms
step:1216/1480 train_time:188052ms step_avg:155.93ms
step:1217/1480 train_time:188215ms step_avg:155.94ms
step:1218/1480 train_time:188377ms step_avg:155.94ms
step:1219/1480 train_time:188547ms step_avg:155.95ms
step:1220/1480 train_time:188710ms step_avg:155.96ms
step:1221/1480 train_time:188872ms step_avg:155.96ms
step:1222/1480 train_time:189032ms step_avg:155.97ms
step:1223/1480 train_time:189195ms step_avg:155.97ms
step:1224/1480 train_time:189359ms step_avg:155.98ms
step:1225/1480 train_time:189523ms step_avg:155.99ms
step:1226/1480 train_time:189689ms step_avg:155.99ms
step:1227/1480 train_time:189854ms step_avg:156.00ms
step:1228/1480 train_time:190016ms step_avg:156.01ms
step:1229/1480 train_time:190178ms step_avg:156.01ms
step:1230/1480 train_time:190347ms step_avg:156.02ms
step:1231/1480 train_time:190513ms step_avg:156.03ms
step:1232/1480 train_time:190677ms step_avg:156.04ms
step:1233/1480 train_time:190838ms step_avg:156.04ms
step:1234/1480 train_time:190998ms step_avg:156.04ms
step:1235/1480 train_time:191165ms step_avg:156.05ms
step:1236/1480 train_time:191328ms step_avg:156.06ms
step:1237/1480 train_time:191491ms step_avg:156.06ms
step:1238/1480 train_time:191662ms step_avg:156.08ms
step:1239/1480 train_time:191824ms step_avg:156.08ms
step:1240/1480 train_time:191990ms step_avg:156.09ms
step:1241/1480 train_time:192154ms step_avg:156.10ms
step:1242/1480 train_time:192316ms step_avg:156.10ms
step:1243/1480 train_time:192478ms step_avg:156.11ms
step:1244/1480 train_time:192638ms step_avg:156.11ms
step:1245/1480 train_time:192801ms step_avg:156.11ms
step:1246/1480 train_time:192965ms step_avg:156.12ms
step:1247/1480 train_time:193127ms step_avg:156.13ms
step:1248/1480 train_time:193289ms step_avg:156.13ms
step:1249/1480 train_time:193452ms step_avg:156.14ms
step:1250/1480 train_time:193614ms step_avg:156.14ms
step:1250/1480 val_loss:3.3380 train_time:193688ms step_avg:156.20ms
step:1251/1480 train_time:193783ms step_avg:156.15ms
step:1252/1480 train_time:193946ms step_avg:156.16ms
step:1253/1480 train_time:194107ms step_avg:156.16ms
step:1254/1480 train_time:194268ms step_avg:156.16ms
step:1255/1480 train_time:194440ms step_avg:156.18ms
step:1256/1480 train_time:194605ms step_avg:156.18ms
step:1257/1480 train_time:194767ms step_avg:156.19ms
step:1258/1480 train_time:194933ms step_avg:156.20ms
step:1259/1480 train_time:195095ms step_avg:156.20ms
step:1260/1480 train_time:195256ms step_avg:156.20ms
step:1261/1480 train_time:195419ms step_avg:156.21ms
step:1262/1480 train_time:195585ms step_avg:156.22ms
step:1263/1480 train_time:195750ms step_avg:156.23ms
step:1264/1480 train_time:195909ms step_avg:156.23ms
step:1265/1480 train_time:196069ms step_avg:156.23ms
step:1266/1480 train_time:196231ms step_avg:156.23ms
step:1267/1480 train_time:196392ms step_avg:156.24ms
step:1268/1480 train_time:196554ms step_avg:156.24ms
step:1269/1480 train_time:196719ms step_avg:156.25ms
step:1270/1480 train_time:196882ms step_avg:156.26ms
step:1271/1480 train_time:197046ms step_avg:156.26ms
step:1272/1480 train_time:197208ms step_avg:156.27ms
step:1273/1480 train_time:197370ms step_avg:156.27ms
step:1274/1480 train_time:197534ms step_avg:156.28ms
step:1275/1480 train_time:197694ms step_avg:156.28ms
step:1276/1480 train_time:197853ms step_avg:156.28ms
step:1277/1480 train_time:198015ms step_avg:156.29ms
step:1278/1480 train_time:198174ms step_avg:156.29ms
step:1279/1480 train_time:198337ms step_avg:156.29ms
step:1280/1480 train_time:198505ms step_avg:156.30ms
step:1281/1480 train_time:198667ms step_avg:156.31ms
step:1282/1480 train_time:198827ms step_avg:156.31ms
step:1283/1480 train_time:198989ms step_avg:156.31ms
step:1284/1480 train_time:199152ms step_avg:156.32ms
step:1285/1480 train_time:199314ms step_avg:156.32ms
step:1286/1480 train_time:199475ms step_avg:156.33ms
step:1287/1480 train_time:199636ms step_avg:156.33ms
step:1288/1480 train_time:199802ms step_avg:156.34ms
step:1289/1480 train_time:199972ms step_avg:156.35ms
step:1290/1480 train_time:200139ms step_avg:156.36ms
step:1291/1480 train_time:200304ms step_avg:156.37ms
step:1292/1480 train_time:200469ms step_avg:156.37ms
step:1293/1480 train_time:200635ms step_avg:156.38ms
step:1294/1480 train_time:200798ms step_avg:156.38ms
step:1295/1480 train_time:200960ms step_avg:156.39ms
step:1296/1480 train_time:201124ms step_avg:156.40ms
step:1297/1480 train_time:201287ms step_avg:156.40ms
step:1298/1480 train_time:201450ms step_avg:156.41ms
step:1299/1480 train_time:201613ms step_avg:156.41ms
step:1300/1480 train_time:201774ms step_avg:156.41ms
step:1301/1480 train_time:201934ms step_avg:156.42ms
step:1302/1480 train_time:202099ms step_avg:156.42ms
step:1303/1480 train_time:202269ms step_avg:156.43ms
step:1304/1480 train_time:202434ms step_avg:156.44ms
step:1305/1480 train_time:202595ms step_avg:156.44ms
step:1306/1480 train_time:202760ms step_avg:156.45ms
step:1307/1480 train_time:202922ms step_avg:156.45ms
step:1308/1480 train_time:203085ms step_avg:156.46ms
step:1309/1480 train_time:203251ms step_avg:156.47ms
step:1310/1480 train_time:203414ms step_avg:156.47ms
step:1311/1480 train_time:203574ms step_avg:156.48ms
step:1312/1480 train_time:203740ms step_avg:156.48ms
step:1313/1480 train_time:203903ms step_avg:156.49ms
step:1314/1480 train_time:204068ms step_avg:156.49ms
step:1315/1480 train_time:204232ms step_avg:156.50ms
step:1316/1480 train_time:204391ms step_avg:156.50ms
step:1317/1480 train_time:204553ms step_avg:156.51ms
step:1318/1480 train_time:204719ms step_avg:156.51ms
step:1319/1480 train_time:204885ms step_avg:156.52ms
step:1320/1480 train_time:205052ms step_avg:156.53ms
step:1321/1480 train_time:205215ms step_avg:156.53ms
step:1322/1480 train_time:205387ms step_avg:156.54ms
step:1323/1480 train_time:205551ms step_avg:156.55ms
step:1324/1480 train_time:205716ms step_avg:156.56ms
step:1325/1480 train_time:205887ms step_avg:156.57ms
step:1326/1480 train_time:206052ms step_avg:156.57ms
step:1327/1480 train_time:206214ms step_avg:156.58ms
step:1328/1480 train_time:206375ms step_avg:156.58ms
step:1329/1480 train_time:206564ms step_avg:156.61ms
step:1330/1480 train_time:206725ms step_avg:156.61ms
step:1331/1480 train_time:206888ms step_avg:156.61ms
step:1332/1480 train_time:207052ms step_avg:156.62ms
step:1333/1480 train_time:207217ms step_avg:156.63ms
step:1334/1480 train_time:207379ms step_avg:156.63ms
step:1335/1480 train_time:207540ms step_avg:156.63ms
step:1336/1480 train_time:207709ms step_avg:156.64ms
step:1337/1480 train_time:207876ms step_avg:156.65ms
step:1338/1480 train_time:208040ms step_avg:156.66ms
step:1339/1480 train_time:208206ms step_avg:156.66ms
step:1340/1480 train_time:208369ms step_avg:156.67ms
step:1341/1480 train_time:208533ms step_avg:156.67ms
step:1342/1480 train_time:208697ms step_avg:156.68ms
step:1343/1480 train_time:208859ms step_avg:156.68ms
step:1344/1480 train_time:209020ms step_avg:156.69ms
step:1345/1480 train_time:209186ms step_avg:156.69ms
step:1346/1480 train_time:209349ms step_avg:156.70ms
step:1347/1480 train_time:209513ms step_avg:156.70ms
step:1348/1480 train_time:209675ms step_avg:156.71ms
step:1349/1480 train_time:209838ms step_avg:156.71ms
step:1350/1480 train_time:210003ms step_avg:156.72ms
step:1351/1480 train_time:210165ms step_avg:156.72ms
step:1352/1480 train_time:210328ms step_avg:156.73ms
step:1353/1480 train_time:210493ms step_avg:156.73ms
step:1354/1480 train_time:210657ms step_avg:156.74ms
step:1355/1480 train_time:210820ms step_avg:156.74ms
step:1356/1480 train_time:210985ms step_avg:156.75ms
step:1357/1480 train_time:211149ms step_avg:156.75ms
step:1358/1480 train_time:211312ms step_avg:156.76ms
step:1359/1480 train_time:211475ms step_avg:156.76ms
step:1360/1480 train_time:211642ms step_avg:156.77ms
step:1361/1480 train_time:211811ms step_avg:156.78ms
step:1362/1480 train_time:211976ms step_avg:156.79ms
step:1363/1480 train_time:212144ms step_avg:156.80ms
step:1364/1480 train_time:212307ms step_avg:156.80ms
step:1365/1480 train_time:212467ms step_avg:156.80ms
step:1366/1480 train_time:212631ms step_avg:156.81ms
step:1367/1480 train_time:212792ms step_avg:156.81ms
step:1368/1480 train_time:212958ms step_avg:156.82ms
step:1369/1480 train_time:213130ms step_avg:156.83ms
step:1370/1480 train_time:213295ms step_avg:156.83ms
step:1371/1480 train_time:213458ms step_avg:156.84ms
step:1372/1480 train_time:213626ms step_avg:156.85ms
step:1373/1480 train_time:213788ms step_avg:156.85ms
step:1374/1480 train_time:213954ms step_avg:156.86ms
step:1375/1480 train_time:214117ms step_avg:156.86ms
step:1375/1480 val_loss:3.2990 train_time:214191ms step_avg:156.92ms
step:1376/1480 train_time:214283ms step_avg:156.87ms
step:1377/1480 train_time:214447ms step_avg:156.87ms
step:1378/1480 train_time:214609ms step_avg:156.88ms
step:1379/1480 train_time:214774ms step_avg:156.88ms
step:1380/1480 train_time:214937ms step_avg:156.89ms
step:1381/1480 train_time:215105ms step_avg:156.90ms
step:1382/1480 train_time:215271ms step_avg:156.90ms
step:1383/1480 train_time:215432ms step_avg:156.91ms
step:1384/1480 train_time:215598ms step_avg:156.91ms
step:1385/1480 train_time:215758ms step_avg:156.92ms
step:1386/1480 train_time:215923ms step_avg:156.92ms
step:1387/1480 train_time:216088ms step_avg:156.93ms
step:1388/1480 train_time:216249ms step_avg:156.93ms
step:1389/1480 train_time:216413ms step_avg:156.93ms
step:1390/1480 train_time:216575ms step_avg:156.94ms
step:1391/1480 train_time:216737ms step_avg:156.94ms
step:1392/1480 train_time:216902ms step_avg:156.95ms
step:1393/1480 train_time:217066ms step_avg:156.95ms
step:1394/1480 train_time:217230ms step_avg:156.96ms
step:1395/1480 train_time:217392ms step_avg:156.96ms
step:1396/1480 train_time:217555ms step_avg:156.97ms
step:1397/1480 train_time:217715ms step_avg:156.97ms
step:1398/1480 train_time:217875ms step_avg:156.97ms
step:1399/1480 train_time:218037ms step_avg:156.97ms
step:1400/1480 train_time:218207ms step_avg:156.98ms
step:1401/1480 train_time:218368ms step_avg:156.99ms
step:1402/1480 train_time:218529ms step_avg:156.99ms
step:1403/1480 train_time:218696ms step_avg:157.00ms
step:1404/1480 train_time:218858ms step_avg:157.00ms
step:1405/1480 train_time:219025ms step_avg:157.01ms
step:1406/1480 train_time:219191ms step_avg:157.01ms
step:1407/1480 train_time:219352ms step_avg:157.02ms
step:1408/1480 train_time:219513ms step_avg:157.02ms
step:1409/1480 train_time:219686ms step_avg:157.03ms
step:1410/1480 train_time:219848ms step_avg:157.03ms
step:1411/1480 train_time:220008ms step_avg:157.04ms
step:1412/1480 train_time:220171ms step_avg:157.04ms
step:1413/1480 train_time:220332ms step_avg:157.04ms
step:1414/1480 train_time:220496ms step_avg:157.05ms
step:1415/1480 train_time:220662ms step_avg:157.05ms
step:1416/1480 train_time:220835ms step_avg:157.07ms
step:1417/1480 train_time:220999ms step_avg:157.07ms
step:1418/1480 train_time:221163ms step_avg:157.08ms
step:1419/1480 train_time:221327ms step_avg:157.08ms
step:1420/1480 train_time:221492ms step_avg:157.09ms
step:1421/1480 train_time:221656ms step_avg:157.09ms
step:1422/1480 train_time:221820ms step_avg:157.10ms
step:1423/1480 train_time:221982ms step_avg:157.10ms
step:1424/1480 train_time:222150ms step_avg:157.11ms
step:1425/1480 train_time:222318ms step_avg:157.12ms
step:1426/1480 train_time:222482ms step_avg:157.12ms
step:1427/1480 train_time:222649ms step_avg:157.13ms
step:1428/1480 train_time:222812ms step_avg:157.13ms
step:1429/1480 train_time:222972ms step_avg:157.13ms
step:1430/1480 train_time:223136ms step_avg:157.14ms
step:1431/1480 train_time:223302ms step_avg:157.14ms
step:1432/1480 train_time:223471ms step_avg:157.15ms
step:1433/1480 train_time:223639ms step_avg:157.16ms
step:1434/1480 train_time:223809ms step_avg:157.17ms
step:1435/1480 train_time:223975ms step_avg:157.18ms
step:1436/1480 train_time:224139ms step_avg:157.18ms
step:1437/1480 train_time:224300ms step_avg:157.18ms
step:1438/1480 train_time:224462ms step_avg:157.19ms
step:1439/1480 train_time:224628ms step_avg:157.19ms
step:1440/1480 train_time:224791ms step_avg:157.20ms
step:1441/1480 train_time:224954ms step_avg:157.20ms
step:1442/1480 train_time:225121ms step_avg:157.21ms
step:1443/1480 train_time:225297ms step_avg:157.22ms
step:1444/1480 train_time:225459ms step_avg:157.22ms
step:1445/1480 train_time:225621ms step_avg:157.23ms
step:1446/1480 train_time:225788ms step_avg:157.23ms
step:1447/1480 train_time:225955ms step_avg:157.24ms
step:1448/1480 train_time:226117ms step_avg:157.24ms
step:1449/1480 train_time:226281ms step_avg:157.25ms
step:1450/1480 train_time:226445ms step_avg:157.25ms
step:1451/1480 train_time:226609ms step_avg:157.26ms
step:1452/1480 train_time:226773ms step_avg:157.26ms
step:1453/1480 train_time:226935ms step_avg:157.27ms
step:1454/1480 train_time:227098ms step_avg:157.27ms
step:1455/1480 train_time:227269ms step_avg:157.28ms
step:1456/1480 train_time:227432ms step_avg:157.28ms
step:1457/1480 train_time:227595ms step_avg:157.29ms
step:1458/1480 train_time:227757ms step_avg:157.29ms
step:1459/1480 train_time:227922ms step_avg:157.30ms
step:1460/1480 train_time:228086ms step_avg:157.30ms
step:1461/1480 train_time:228251ms step_avg:157.31ms
step:1462/1480 train_time:228414ms step_avg:157.31ms
step:1463/1480 train_time:228579ms step_avg:157.32ms
step:1464/1480 train_time:228748ms step_avg:157.32ms
step:1465/1480 train_time:228913ms step_avg:157.33ms
step:1466/1480 train_time:229074ms step_avg:157.33ms
step:1467/1480 train_time:229237ms step_avg:157.34ms
step:1468/1480 train_time:229402ms step_avg:157.34ms
step:1469/1480 train_time:229566ms step_avg:157.34ms
step:1470/1480 train_time:229733ms step_avg:157.35ms
step:1471/1480 train_time:229903ms step_avg:157.36ms
step:1472/1480 train_time:230074ms step_avg:157.37ms
step:1473/1480 train_time:230237ms step_avg:157.37ms
step:1474/1480 train_time:230405ms step_avg:157.38ms
step:1475/1480 train_time:230575ms step_avg:157.39ms
step:1476/1480 train_time:230737ms step_avg:157.39ms
step:1477/1480 train_time:230906ms step_avg:157.40ms
step:1478/1480 train_time:231076ms step_avg:157.41ms
step:1479/1480 train_time:231239ms step_avg:157.41ms
step:1480/1480 train_time:231401ms step_avg:157.42ms
step:1480/1480 val_loss:3.2798 train_time:231478ms step_avg:157.47ms
peak memory consumption: 34239 MiB