7442abb4-a571-4340-9844-de16209c3762.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
import contextlib
from dataclasses import dataclass
from pathlib import Path

import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.nn.attention.flex_attention import BlockMask, flex_attention #KoszarskyB

# -----------------------------------------------------------------------------
# Muon optimizer

@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert len(G.shape) == 2
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    X /= (X.norm() + eps) # ensure top singular value <= 1
    if G.size(0) > G.size(1):
        X = X.T
    for _ in range(steps):
        A = X @ X.T
        B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    if G.size(0) > G.size(1):
        X = X.T
    return X

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven't tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        ns_steps: The number of Newton-Schulz iteration steps to use.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True, ns_steps=5):
        self.world_size = int(os.environ['WORLD_SIZE'])
        self.rank = int(os.environ['RANK'])
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, ns_steps=ns_steps)
        params = list(params)
        assert all(isinstance(p, torch.Tensor) for p in params)
        sizes = {p.numel() for p in params}
        param_groups = [
            {
                'params': [p for p in params if p.numel() == size],
                'update_buffer': [
                    torch.empty(size, device='cuda', dtype=torch.bfloat16)
                    for _ in range(self.world_size)
                ],
            }
            for size in sizes
        ]
        super().__init__(param_groups, defaults)

    def step(self):

        for group in self.param_groups:

            lr = group['lr']
            momentum = group['momentum']
            nesterov = group['nesterov']
            ns_steps = group['ns_steps']
            update_buffers = group['update_buffer']
            # generate weight updates in distributed fashion
            params = group['params']
            assert len(params) % self.world_size == 0
            handle = None
            params_world = None
            def update_prev():
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffers):
                    p_world.data.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.world_size]:
                p = params[base_i + self.rank]
                g = p.grad
                assert g is not None
                state = self.state[p]
                if 'momentum_buffer' not in state:
                    state['momentum_buffer'] = torch.zeros_like(g)
                buf = state['momentum_buffer']
                buf.lerp_(g, 1 - momentum)
                g = g.lerp_(buf, momentum) if nesterov else buf
                g = zeropower_via_newtonschulz5(g, steps=ns_steps).flatten()
                update_prev()
                handle = dist.all_gather(update_buffers, g, async_op=True)
                params_world = params[base_i : base_i + self.world_size]
            update_prev()

# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):

    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=False)

    def forward(self, x):
        return F.linear(x, self.weight.to(x.dtype))

class Rotary(torch.nn.Module):

    def __init__(self, dim, base=10000):
        super().__init__()
        self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None

    def forward(self, x):
        seq_len = x.shape[1]
        if seq_len != self.seq_len_cached:
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.outer(t, self.inv_freq)
            self.seq_len_cached = seq_len
            self.cos_cached = freqs.cos()
            self.sin_cached = freqs.sin()
        cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
        # apply_rotary_emb(x, cos, sin)
        x1, x2 = x.chunk(2, dim=3)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x)

class CausalSelfAttention(nn.Module):

    def __init__(self, dim, num_heads):
        super().__init__()
        assert dim % num_heads == 0
        self.num_heads = num_heads
        self.c_q = CastedLinear(dim, dim)
        self.c_k = CastedLinear(dim, dim)
        self.c_v = CastedLinear(dim, dim)
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5]))
        self.rotary = Rotary(dim // num_heads) # dim // num_heads = head_dim
        self.c_proj = CastedLinear(dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x, vi, block_mask):
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q = self.c_q(x).view(B, T, self.num_heads, -1)
        k = self.c_k(x).view(B, T, self.num_heads, -1)
        v = self.c_v(x).view(B, T, self.num_heads, -1)
        v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @KoszarskyB & @Grad62304977
        q, k = norm(q), norm(k) # QK norm @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask, enable_gqa=True)
        y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):

    def __init__(self, dim):
        super().__init__()
        self.c_fc   = CastedLinear(dim, 4 * dim)
        self.c_proj = CastedLinear(4 * dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x):
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):

    def __init__(self, config):
        super().__init__()
        self.attn = CausalSelfAttention(config.model_dim, config.num_heads)
        self.mlp = MLP(config.model_dim)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x, vi, x0, block_mask):
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        x = x + self.attn(norm(x), vi, block_mask)
        x = x + self.mlp(norm(x))
        return x

class ValueEmbedding(nn.Module):
    def __init__(self, config: "GPTConfig"):
        super().__init__()
        self.__setattr__
        self.embed = nn.ModuleList([
            nn.Embedding(config.vocab_size, config.model_dim)
            for _ in range(6)
        ])

    def forward(self, inputs) -> "list[torch.Tensor]":
        ve = [emb(inputs) for emb in self.embed]
        ve += reversed(ve)
        return ve


# -----------------------------------------------------------------------------
# The main GPT-2 model

@dataclass
class GPTConfig:
    vocab_size : int = 50304
    num_layers : int = 12
    num_heads : int = 6 # head dim 128 suggested by @Grad62304977
    model_dim : int = 768

class GPT(nn.Module):

    def __init__(self, config: GPTConfig):
        super().__init__()
        self.num_layers = config.num_layers

        # U-net design by @brendanh0gan
        self.num_encoder_layers = config.num_layers // 2 # Half of the layers for encoder
        self.num_decoder_layers = config.num_layers - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))

        self.embed = nn.Embedding(config.vocab_size, config.model_dim)
        self.blocks = nn.ModuleList([Block(config) for _ in range(config.num_layers)])
        # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
        # U-net structure on token value embeddings by @leloykun
        self.value_embeds = ValueEmbedding(config)
        self.lm_head = CastedLinear(config.model_dim, config.vocab_size)
        self.lm_head.weight.data.zero_() # @Grad62304977

    def forward(
        self,
        inputs: torch.Tensor,
        targets: torch.Tensor,
        sliding_window_num_blocks: torch.Tensor,
    ):
        BLOCK_SIZE = 128
        assert inputs.ndim == 1
        docs = (inputs == 50256).cumsum(0)
        docs_low = docs.view(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.view(-1, BLOCK_SIZE)[:, -1].contiguous()

        def document_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            return causal_mask & document_mask

        def dense_to_ordered(dense_mask: torch.Tensor):
            num_blocks = dense_mask.sum(dim=-1, dtype=torch.int32)
            indices = dense_mask.argsort(dim=-1, descending=True, stable=True).to(torch.int32)
            return num_blocks[None, None].contiguous(), indices[None, None].contiguous()

        def create_doc_swc_block_mask(sliding_window_num_blocks: torch.Tensor):
            kv_idx = block_idx = torch.arange(512, dtype=torch.int32, device="cuda")
            q_idx = block_idx[:, None]
            causal_bm = q_idx >= kv_idx
            causal_full_bm = q_idx > kv_idx
            window_bm = q_idx - kv_idx < sliding_window_num_blocks
            window_full_bm = window_bm
            # document_bm = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
            document_bm = (docs_low[:, None] <= docs_high) & (docs_low <= docs_high[:, None])
            document_full_bm = (docs_low[:, None] == docs_high) & (docs_low == docs_high[:, None])
            nonzero_bm = causal_bm & window_bm & document_bm
            full_bm  = causal_full_bm & window_full_bm & document_full_bm
            kv_num_blocks, kv_indices = dense_to_ordered(nonzero_bm ^ full_bm)
            full_kv_num_blocks, full_kv_indices = dense_to_ordered(full_bm)
            return BlockMask.from_kv_blocks(
                kv_num_blocks,
                kv_indices,
                full_kv_num_blocks,
                full_kv_indices,
                BLOCK_SIZE=BLOCK_SIZE,
                mask_mod=document_causal,
            )

        block_mask = create_doc_swc_block_mask(sliding_window_num_blocks)

        # forward the GPT model itself
        x = self.embed(inputs[None]) # token embeddings of shape (b, t, model_dim)
        x = norm(x) # @Grad62304977
        x0 = x
        ve = self.value_embeds(inputs)
        ve_enc, ve_dec = ve[:self.num_encoder_layers], ve[self.num_encoder_layers:]

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        for i in range(self.num_encoder_layers):
            x = self.blocks[i](x, ve_enc[i], x0, block_mask)
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            # U-net structure on token value embeddings by @leloykun
            x = self.blocks[self.num_encoder_layers + i](x, ve_dec[i], x0, block_mask)

        x = norm(x)
        logits = self.lm_head(x)
        logits = 30 * torch.tanh(logits / 30) # @Grad62304977
        logits = logits.float()
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1))
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _peek_data_shard(file: Path):
    # only reads the header, returns header data
    # header is 256 int32
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    return int(header[2]) # number of tokens (claimed)

def _load_data_shard(path: Path, num_tokens):
    with path.open("rb", buffering=0) as f:
        tokens = torch.empty(num_tokens, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy())
        assert nbytes == 2 * num_tokens, "number of tokens read does not match header?"
    return tokens

class DistributedDataLoader:
    def __init__(self, filename_pattern, seq_len, process_rank, num_processes):
        self.process_rank = process_rank
        self.num_processes = num_processes
        self.seq_len = seq_len

        # glob files that match the pattern
        self.files = sorted(Path.cwd().glob(filename_pattern))
        assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"

        # load and validate all data shards, count number of tokens in total
        self.files_num_tokens = [_peek_data_shard(file) for file in self.files]
        assert min(self.files_num_tokens) >= num_processes * seq_len + 1
        self.total_num_tokens = sum(self.files_num_tokens)

        self.reset()

    def reset(self):
        self.current_shard = -1
        self.advance()

    def advance(self): # advance to next data shard
        self.current_shard = (self.current_shard + 1) % len(self.files)
        self.current_position = self.process_rank * self.seq_len
        self.tokens = _load_data_shard(self.files[self.current_shard], self.files_num_tokens[self.current_shard])

    def next_batch(self):
        batch_size = self.seq_len * self.num_processes
        buf = self.tokens[self.current_position:self.current_position+self.seq_len+1]
        # host side async is sufficient;
        # no performance improvement was observed when introducing a separate stream.
        inputs = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
        targets = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
        # advance current position and load next shard if necessary
        self.current_position += batch_size
        if self.current_position + batch_size + 1 >= len(self.tokens):
            self.advance()
        return inputs, targets

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data hyperparams
    input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
    input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
    # optimization hyperparams
    batch_size : int = 8 # batch size, in sequences, across all devices
    sequence_length : int = 64*1024 # sequence length, in tokens
    num_iterations : int = 1480 # number of iterations to run
    warmup_iters : int = 0
    cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
    weight_decay : float = 0
    # evaluation and logging hyperparams
    val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
    val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()

# set up DDP (distributed data parallel). torchrun sets this env variable
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
assert torch.cuda.is_available()
device = torch.device(f"cuda:{ddp_local_rank}")
torch.cuda.set_device(device)
print(f"using device: {device}")
dist.init_process_group(backend='nccl', device_id=device)
dist.barrier()
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = uuid.uuid4()
    logdir = Path("logs") / f"{run_id}"
    logdir.mkdir(exist_ok=True)
    logfile = Path("logs") / f"{run_id}.txt"
    print(logfile.stem)
    # create the log file
    with logfile.open("w") as f:
        # begin the log by printing this file (the Python code)
        print(code, file=f)
        print("=" * 100, file=f)
def print0(s, logonly=False):
    if master_process:
        with logfile.open("a") as f:
            if not logonly:
                print(s)
            print(s, file=f)
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running python {sys.version}")
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)

# calculate the number of steps to take in the val loop.
assert args.val_tokens % (args.sequence_length * ddp_world_size) == 0
val_steps = args.val_tokens // (args.sequence_length * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size

# load tokens
train_loader = DistributedDataLoader(args.input_bin, args.sequence_length, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, args.sequence_length, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.total_num_tokens} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.total_num_tokens} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
inputs_train, targets_train = train_loader.next_batch()

# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, num_layers=12, num_heads=6, model_dim=768))
model = model.cuda().bfloat16()
for m in model.modules():
    if isinstance(m, CastedLinear):
        m.float()
config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank], broadcast_buffers=False, gradient_as_bucket_view=True)
raw_model = model.module # always contains the "raw" unwrapped model

# init the optimizer(s)
embed_params = [*raw_model.embed.parameters(), *raw_model.value_embeds.parameters()]
optimizer1 = torch.optim.Adam(embed_params, lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.blocks.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
    assert it <= args.num_iterations
    # 1) linear warmup for warmup_iters steps
    if it < args.warmup_iters:
        return (it+1) / args.warmup_iters
    # 2) constant lr for a while
    elif it < args.num_iterations - args.cooldown_iters:
        return 1.0
    # 3) linear cooldown
    else:
        decay_ratio = (args.num_iterations - it) / args.cooldown_iters
        return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]

sliding_window_num_blocks = torch.tensor(1, dtype=torch.int32, device="cuda")
sw_num_blocks_prev = 1
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
    last_step = (step == args.num_iterations)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Linearly increase the sliding window size over training in chunks of 64 from 64 -> 1792. By @fernbear.bsky.social
    frac_done = step / args.num_iterations # training progress
    sw_num_blocks = int(((1 - frac_done) * 64 + frac_done * 1792 + 64) // 128)
    if sw_num_blocks != sw_num_blocks_prev:
        sliding_window_num_blocks.copy_(sw_num_blocks, non_blocking=True)
        sw_num_blocks_prev = sw_num_blocks

    # once in a while evaluate the validation dataset
    if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # run validation batches
        model.eval()
        val_loader.reset()
        val_loss = 0.0
        for _ in range(val_steps):
            with torch.no_grad():
                inputs_val, targets_val = val_loader.next_batch()
                val_loss += model(inputs_val, targets_val, sliding_window_num_blocks)
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        val_loss /= val_steps
        # log val loss to console and to logfile
        print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # save the state of the training process
        log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
        torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    # bit confusing: we want to make sure to eval on 0th iteration
    # but also after the very last iteration. so we loop for step <= num_iterations
    # instead of just < num_iterations (one extra due to <=), only to do
    # the validation/sampling one last time, and then we break right here as we're done.
    if last_step:
        break

    # --------------- TRAINING SECTION BEGIN -----------------
    model.train()
    for i in range(1, train_accumulation_steps + 1):
        with contextlib.ExitStack() as stack:
            if i < train_accumulation_steps: # there's no need to sync gradients every accumulation step
                stack.enter_context(model.no_sync())
            if step >= 5:
                stack.enter_context(torch.compiler.set_stance(skip_guard_eval_unsafe=True))
            model(inputs_train, targets_train, sliding_window_num_blocks).backward()
            inputs_train, targets_train = train_loader.next_batch()
    if train_accumulation_steps != 1:
        for p in model.parameters():
            p.grad /= train_accumulation_steps
    # momentum warmup for Muon
    frac = min(step/300, 1)
    for group in optimizer3.param_groups:
        group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # --------------- TRAINING SECTION END -------------------
    # everything that follows now is just diagnostics, prints, logging, etc.
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")

print0(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")

# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()

====================================================================================================
Running python 3.10.12 (main, Sep 11 2024, 15:47:36) [GCC 11.4.0]
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Wed Dec 11 07:31:51 2024       
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.183.06             Driver Version: 535.183.06   CUDA Version: 12.2     |
|-----------------------------------------+----------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
|                                         |                      |               MIG M. |
|=========================================+======================+======================|
|   0  NVIDIA H100 80GB HBM3          On  | 00000000:19:00.0 Off |                    0 |
| N/A   38C    P0             125W / 700W |   7084MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  | 00000000:3B:00.0 Off |                    0 |
| N/A   30C    P0             116W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  | 00000000:4C:00.0 Off |                    0 |
| N/A   28C    P0             112W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  | 00000000:5D:00.0 Off |                    0 |
| N/A   36C    P0             114W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  | 00000000:9B:00.0 Off |                    0 |
| N/A   38C    P0             119W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  | 00000000:BB:00.0 Off |                    0 |
| N/A   30C    P0             117W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  | 00000000:CB:00.0 Off |                    0 |
| N/A   36C    P0             119W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  | 00000000:DB:00.0 Off |                    0 |
| N/A   30C    P0             118W / 700W |   3211MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
                                                                                         
+---------------------------------------------------------------------------------------+
| Processes:                                                                            |
|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
|        ID   ID                                                             Usage      |
|=======================================================================================|
+---------------------------------------------------------------------------------------+

====================================================================================================
Training DataLoader: total number of tokens: 1000000000 across 10 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:29159ms step_avg:nanms
step:2/1480 train_time:29264ms step_avg:nanms
step:3/1480 train_time:29387ms step_avg:nanms
step:4/1480 train_time:29529ms step_avg:nanms
step:5/1480 train_time:29669ms step_avg:nanms
step:6/1480 train_time:29811ms step_avg:nanms
step:7/1480 train_time:29951ms step_avg:nanms
step:8/1480 train_time:30093ms step_avg:nanms
step:9/1480 train_time:30237ms step_avg:nanms
step:10/1480 train_time:30381ms step_avg:nanms
step:11/1480 train_time:142ms step_avg:nanms
step:12/1480 train_time:282ms step_avg:nanms
step:13/1480 train_time:425ms step_avg:141.69ms
step:14/1480 train_time:565ms step_avg:141.33ms
step:15/1480 train_time:707ms step_avg:141.46ms
step:16/1480 train_time:849ms step_avg:141.45ms
step:17/1480 train_time:992ms step_avg:141.70ms
step:18/1480 train_time:1134ms step_avg:141.74ms
step:19/1480 train_time:1276ms step_avg:141.76ms
step:20/1480 train_time:1419ms step_avg:141.89ms
step:21/1480 train_time:1562ms step_avg:142.00ms
step:22/1480 train_time:1704ms step_avg:142.03ms
step:23/1480 train_time:1845ms step_avg:141.94ms
step:24/1480 train_time:1987ms step_avg:141.91ms
step:25/1480 train_time:2129ms step_avg:141.96ms
step:26/1480 train_time:2273ms step_avg:142.04ms
step:27/1480 train_time:2416ms step_avg:142.11ms
step:28/1480 train_time:2560ms step_avg:142.22ms
step:29/1480 train_time:2704ms step_avg:142.29ms
step:30/1480 train_time:2845ms step_avg:142.27ms
step:31/1480 train_time:2987ms step_avg:142.25ms
step:32/1480 train_time:3129ms step_avg:142.24ms
step:33/1480 train_time:3272ms step_avg:142.27ms
step:34/1480 train_time:3416ms step_avg:142.34ms
step:35/1480 train_time:3562ms step_avg:142.49ms
step:36/1480 train_time:3705ms step_avg:142.48ms
step:37/1480 train_time:3846ms step_avg:142.45ms
step:38/1480 train_time:3988ms step_avg:142.43ms
step:39/1480 train_time:4519ms step_avg:155.81ms
step:40/1480 train_time:5023ms step_avg:167.43ms
step:41/1480 train_time:5125ms step_avg:165.33ms
step:42/1480 train_time:5267ms step_avg:164.58ms
step:43/1480 train_time:5407ms step_avg:163.86ms
step:44/1480 train_time:5549ms step_avg:163.20ms
step:45/1480 train_time:5691ms step_avg:162.59ms
step:46/1480 train_time:5834ms step_avg:162.06ms
step:47/1480 train_time:5978ms step_avg:161.57ms
step:48/1480 train_time:6125ms step_avg:161.20ms
step:49/1480 train_time:6267ms step_avg:160.69ms
step:50/1480 train_time:6410ms step_avg:160.24ms
step:51/1480 train_time:6552ms step_avg:159.79ms
step:52/1480 train_time:6695ms step_avg:159.40ms
step:53/1480 train_time:6837ms step_avg:159.00ms
step:54/1480 train_time:6980ms step_avg:158.64ms
step:55/1480 train_time:7124ms step_avg:158.31ms
step:56/1480 train_time:7266ms step_avg:157.96ms
step:57/1480 train_time:7408ms step_avg:157.61ms
step:58/1480 train_time:7549ms step_avg:157.27ms
step:59/1480 train_time:7692ms step_avg:156.97ms
step:60/1480 train_time:7835ms step_avg:156.69ms
step:61/1480 train_time:7979ms step_avg:156.45ms
step:62/1480 train_time:8121ms step_avg:156.18ms
step:63/1480 train_time:8264ms step_avg:155.93ms
step:64/1480 train_time:8406ms step_avg:155.67ms
step:65/1480 train_time:8547ms step_avg:155.40ms
step:66/1480 train_time:8690ms step_avg:155.19ms
step:67/1480 train_time:8834ms step_avg:154.98ms
step:68/1480 train_time:8978ms step_avg:154.79ms
step:69/1480 train_time:9122ms step_avg:154.62ms
step:70/1480 train_time:9265ms step_avg:154.41ms
step:71/1480 train_time:9407ms step_avg:154.21ms
step:72/1480 train_time:9548ms step_avg:153.99ms
step:73/1480 train_time:9689ms step_avg:153.80ms
step:74/1480 train_time:9831ms step_avg:153.62ms
step:75/1480 train_time:9976ms step_avg:153.47ms
step:76/1480 train_time:10119ms step_avg:153.32ms
step:77/1480 train_time:10261ms step_avg:153.15ms
step:78/1480 train_time:10404ms step_avg:153.00ms
step:79/1480 train_time:10546ms step_avg:152.84ms
step:80/1480 train_time:10687ms step_avg:152.67ms
step:81/1480 train_time:10829ms step_avg:152.52ms
step:82/1480 train_time:10972ms step_avg:152.39ms
step:83/1480 train_time:11116ms step_avg:152.27ms
step:84/1480 train_time:11260ms step_avg:152.16ms
step:85/1480 train_time:11402ms step_avg:152.03ms
step:86/1480 train_time:11545ms step_avg:151.91ms
step:87/1480 train_time:11686ms step_avg:151.76ms
step:88/1480 train_time:11828ms step_avg:151.64ms
step:89/1480 train_time:11970ms step_avg:151.52ms
step:90/1480 train_time:12113ms step_avg:151.42ms
step:91/1480 train_time:12259ms step_avg:151.35ms
step:92/1480 train_time:12403ms step_avg:151.25ms
step:93/1480 train_time:12546ms step_avg:151.16ms
step:94/1480 train_time:12688ms step_avg:151.05ms
step:95/1480 train_time:12829ms step_avg:150.93ms
step:96/1480 train_time:12972ms step_avg:150.84ms
step:97/1480 train_time:13117ms step_avg:150.76ms
step:98/1480 train_time:13262ms step_avg:150.70ms
step:99/1480 train_time:13406ms step_avg:150.63ms
step:100/1480 train_time:13548ms step_avg:150.53ms
step:101/1480 train_time:13693ms step_avg:150.47ms
step:102/1480 train_time:13832ms step_avg:150.35ms
step:103/1480 train_time:13977ms step_avg:150.29ms
step:104/1480 train_time:14120ms step_avg:150.21ms
step:105/1480 train_time:14264ms step_avg:150.15ms
step:106/1480 train_time:14406ms step_avg:150.06ms
step:107/1480 train_time:14548ms step_avg:149.98ms
step:108/1480 train_time:14691ms step_avg:149.90ms
step:109/1480 train_time:14833ms step_avg:149.83ms
step:110/1480 train_time:14976ms step_avg:149.76ms
step:111/1480 train_time:15121ms step_avg:149.71ms
step:112/1480 train_time:15266ms step_avg:149.67ms
step:113/1480 train_time:15411ms step_avg:149.63ms
step:114/1480 train_time:15557ms step_avg:149.59ms
step:115/1480 train_time:15703ms step_avg:149.55ms
step:116/1480 train_time:15848ms step_avg:149.51ms
step:117/1480 train_time:15993ms step_avg:149.47ms
step:118/1480 train_time:16140ms step_avg:149.44ms
step:119/1480 train_time:16285ms step_avg:149.41ms
step:120/1480 train_time:16430ms step_avg:149.36ms
step:121/1480 train_time:16576ms step_avg:149.33ms
step:122/1480 train_time:16722ms step_avg:149.31ms
step:123/1480 train_time:16867ms step_avg:149.26ms
step:124/1480 train_time:17012ms step_avg:149.22ms
step:125/1480 train_time:17158ms step_avg:149.20ms
step:125/1480 val_loss:4.4345 train_time:17224ms step_avg:149.77ms
step:126/1480 train_time:17319ms step_avg:149.30ms
step:127/1480 train_time:17464ms step_avg:149.27ms
step:128/1480 train_time:17609ms step_avg:149.23ms
step:129/1480 train_time:17755ms step_avg:149.20ms
step:130/1480 train_time:17901ms step_avg:149.17ms
step:131/1480 train_time:18046ms step_avg:149.14ms
step:132/1480 train_time:18191ms step_avg:149.11ms
step:133/1480 train_time:18338ms step_avg:149.09ms
step:134/1480 train_time:18484ms step_avg:149.07ms
step:135/1480 train_time:18630ms step_avg:149.04ms
step:136/1480 train_time:18777ms step_avg:149.02ms
step:137/1480 train_time:18922ms step_avg:148.99ms
step:138/1480 train_time:19066ms step_avg:148.95ms
step:139/1480 train_time:19211ms step_avg:148.92ms
step:140/1480 train_time:19358ms step_avg:148.91ms
step:141/1480 train_time:19502ms step_avg:148.87ms
step:142/1480 train_time:19649ms step_avg:148.85ms
step:143/1480 train_time:19795ms step_avg:148.83ms
step:144/1480 train_time:19941ms step_avg:148.82ms
step:145/1480 train_time:20086ms step_avg:148.78ms
step:146/1480 train_time:20231ms step_avg:148.76ms
step:147/1480 train_time:20379ms step_avg:148.75ms
step:148/1480 train_time:20523ms step_avg:148.72ms
step:149/1480 train_time:20669ms step_avg:148.70ms
step:150/1480 train_time:20816ms step_avg:148.68ms
step:151/1480 train_time:20962ms step_avg:148.67ms
step:152/1480 train_time:21107ms step_avg:148.64ms
step:153/1480 train_time:21253ms step_avg:148.62ms
step:154/1480 train_time:21399ms step_avg:148.60ms
step:155/1480 train_time:21544ms step_avg:148.58ms
step:156/1480 train_time:21690ms step_avg:148.56ms
step:157/1480 train_time:21838ms step_avg:148.56ms
step:158/1480 train_time:21983ms step_avg:148.54ms
step:159/1480 train_time:22128ms step_avg:148.51ms
step:160/1480 train_time:22274ms step_avg:148.49ms
step:161/1480 train_time:22420ms step_avg:148.48ms
step:162/1480 train_time:22565ms step_avg:148.46ms
step:163/1480 train_time:22711ms step_avg:148.44ms
step:164/1480 train_time:22858ms step_avg:148.43ms
step:165/1480 train_time:23003ms step_avg:148.41ms
step:166/1480 train_time:23149ms step_avg:148.39ms
step:167/1480 train_time:23294ms step_avg:148.37ms
step:168/1480 train_time:23440ms step_avg:148.36ms
step:169/1480 train_time:23585ms step_avg:148.33ms
step:170/1480 train_time:23731ms step_avg:148.32ms
step:171/1480 train_time:23877ms step_avg:148.31ms
step:172/1480 train_time:24023ms step_avg:148.29ms
step:173/1480 train_time:24168ms step_avg:148.27ms
step:174/1480 train_time:24314ms step_avg:148.26ms
step:175/1480 train_time:24461ms step_avg:148.25ms
step:176/1480 train_time:24605ms step_avg:148.22ms
step:177/1480 train_time:24750ms step_avg:148.21ms
step:178/1480 train_time:25278ms step_avg:150.46ms
step:179/1480 train_time:25782ms step_avg:152.55ms
step:180/1480 train_time:25887ms step_avg:152.27ms
step:181/1480 train_time:26032ms step_avg:152.24ms
step:182/1480 train_time:26179ms step_avg:152.21ms
step:183/1480 train_time:26324ms step_avg:152.16ms
step:184/1480 train_time:26468ms step_avg:152.12ms
step:185/1480 train_time:26614ms step_avg:152.08ms
step:186/1480 train_time:26761ms step_avg:152.05ms
step:187/1480 train_time:26907ms step_avg:152.02ms
step:188/1480 train_time:27053ms step_avg:151.98ms
step:189/1480 train_time:27214ms step_avg:152.03ms
step:190/1480 train_time:27344ms step_avg:151.91ms
step:191/1480 train_time:27488ms step_avg:151.87ms
step:192/1480 train_time:27634ms step_avg:151.83ms
step:193/1480 train_time:27780ms step_avg:151.80ms
step:194/1480 train_time:27925ms step_avg:151.77ms
step:195/1480 train_time:28070ms step_avg:151.73ms
step:196/1480 train_time:28217ms step_avg:151.70ms
step:197/1480 train_time:28363ms step_avg:151.67ms
step:198/1480 train_time:28507ms step_avg:151.63ms
step:199/1480 train_time:28653ms step_avg:151.60ms
step:200/1480 train_time:28799ms step_avg:151.57ms
step:201/1480 train_time:28948ms step_avg:151.56ms
step:202/1480 train_time:29091ms step_avg:151.51ms
step:203/1480 train_time:29238ms step_avg:151.49ms
step:204/1480 train_time:29383ms step_avg:151.46ms
step:205/1480 train_time:29528ms step_avg:151.42ms
step:206/1480 train_time:29673ms step_avg:151.39ms
step:207/1480 train_time:29820ms step_avg:151.37ms
step:208/1480 train_time:29965ms step_avg:151.34ms
step:209/1480 train_time:30110ms step_avg:151.31ms
step:210/1480 train_time:30258ms step_avg:151.29ms
step:211/1480 train_time:30402ms step_avg:151.25ms
step:212/1480 train_time:30548ms step_avg:151.23ms
step:213/1480 train_time:30694ms step_avg:151.20ms
step:214/1480 train_time:30841ms step_avg:151.18ms
step:215/1480 train_time:30985ms step_avg:151.15ms
step:216/1480 train_time:31131ms step_avg:151.12ms
step:217/1480 train_time:31277ms step_avg:151.10ms
step:218/1480 train_time:31423ms step_avg:151.07ms
step:219/1480 train_time:31567ms step_avg:151.04ms
step:220/1480 train_time:31713ms step_avg:151.02ms
step:221/1480 train_time:32250ms step_avg:152.84ms
step:222/1480 train_time:32757ms step_avg:154.51ms
step:223/1480 train_time:32865ms step_avg:154.29ms
step:224/1480 train_time:33013ms step_avg:154.27ms
step:225/1480 train_time:33162ms step_avg:154.24ms
step:226/1480 train_time:33310ms step_avg:154.21ms
step:227/1480 train_time:33459ms step_avg:154.19ms
step:228/1480 train_time:33606ms step_avg:154.16ms
step:229/1480 train_time:33755ms step_avg:154.13ms
step:230/1480 train_time:33903ms step_avg:154.11ms
step:231/1480 train_time:34053ms step_avg:154.08ms
step:232/1480 train_time:34202ms step_avg:154.06ms
step:233/1480 train_time:34351ms step_avg:154.04ms
step:234/1480 train_time:34500ms step_avg:154.02ms
step:235/1480 train_time:34648ms step_avg:153.99ms
step:236/1480 train_time:34798ms step_avg:153.97ms
step:237/1480 train_time:34945ms step_avg:153.94ms
step:238/1480 train_time:35094ms step_avg:153.92ms
step:239/1480 train_time:35243ms step_avg:153.90ms
step:240/1480 train_time:35391ms step_avg:153.87ms
step:241/1480 train_time:35540ms step_avg:153.85ms
step:242/1480 train_time:35688ms step_avg:153.83ms
step:243/1480 train_time:35836ms step_avg:153.80ms
step:244/1480 train_time:35985ms step_avg:153.78ms
step:245/1480 train_time:36134ms step_avg:153.76ms
step:246/1480 train_time:36283ms step_avg:153.74ms
step:247/1480 train_time:36430ms step_avg:153.71ms
step:248/1480 train_time:36580ms step_avg:153.70ms
step:249/1480 train_time:36728ms step_avg:153.67ms
step:250/1480 train_time:36876ms step_avg:153.65ms
step:250/1480 val_loss:3.9911 train_time:36942ms step_avg:153.93ms
step:251/1480 train_time:37036ms step_avg:153.68ms
step:252/1480 train_time:37182ms step_avg:153.64ms
step:253/1480 train_time:37331ms step_avg:153.63ms
step:254/1480 train_time:37479ms step_avg:153.60ms
step:255/1480 train_time:37626ms step_avg:153.58ms
step:256/1480 train_time:37775ms step_avg:153.56ms
step:257/1480 train_time:37922ms step_avg:153.53ms
step:258/1480 train_time:38072ms step_avg:153.51ms
step:259/1480 train_time:38220ms step_avg:153.49ms
step:260/1480 train_time:38369ms step_avg:153.48ms
step:261/1480 train_time:38518ms step_avg:153.46ms
step:262/1480 train_time:38666ms step_avg:153.44ms
step:263/1480 train_time:38815ms step_avg:153.42ms
step:264/1480 train_time:38963ms step_avg:153.40ms
step:265/1480 train_time:39111ms step_avg:153.37ms
step:266/1480 train_time:39259ms step_avg:153.36ms
step:267/1480 train_time:39407ms step_avg:153.33ms
step:268/1480 train_time:39556ms step_avg:153.32ms
step:269/1480 train_time:39704ms step_avg:153.30ms
step:270/1480 train_time:39853ms step_avg:153.28ms
step:271/1480 train_time:40001ms step_avg:153.26ms
step:272/1480 train_time:40150ms step_avg:153.25ms
step:273/1480 train_time:40299ms step_avg:153.23ms
step:274/1480 train_time:40447ms step_avg:153.21ms
step:275/1480 train_time:40596ms step_avg:153.19ms
step:276/1480 train_time:40744ms step_avg:153.17ms
step:277/1480 train_time:40893ms step_avg:153.16ms
step:278/1480 train_time:41040ms step_avg:153.14ms
step:279/1480 train_time:41189ms step_avg:153.12ms
step:280/1480 train_time:41338ms step_avg:153.10ms
step:281/1480 train_time:41487ms step_avg:153.09ms
step:282/1480 train_time:41636ms step_avg:153.07ms
step:283/1480 train_time:41785ms step_avg:153.06ms
step:284/1480 train_time:41933ms step_avg:153.04ms
step:285/1480 train_time:42081ms step_avg:153.02ms
step:286/1480 train_time:42231ms step_avg:153.01ms
step:287/1480 train_time:42379ms step_avg:152.99ms
step:288/1480 train_time:42528ms step_avg:152.98ms
step:289/1480 train_time:42677ms step_avg:152.96ms
step:290/1480 train_time:42825ms step_avg:152.95ms
step:291/1480 train_time:42974ms step_avg:152.93ms
step:292/1480 train_time:43121ms step_avg:152.91ms
step:293/1480 train_time:43270ms step_avg:152.90ms
step:294/1480 train_time:43418ms step_avg:152.88ms
step:295/1480 train_time:43567ms step_avg:152.87ms
step:296/1480 train_time:43717ms step_avg:152.85ms
step:297/1480 train_time:43865ms step_avg:152.84ms
step:298/1480 train_time:44014ms step_avg:152.83ms
step:299/1480 train_time:44162ms step_avg:152.81ms
step:300/1480 train_time:44311ms step_avg:152.80ms
step:301/1480 train_time:44459ms step_avg:152.78ms
step:302/1480 train_time:44607ms step_avg:152.76ms
step:303/1480 train_time:44756ms step_avg:152.75ms
step:304/1480 train_time:44904ms step_avg:152.73ms
step:305/1480 train_time:45053ms step_avg:152.72ms
step:306/1480 train_time:45201ms step_avg:152.71ms
step:307/1480 train_time:45350ms step_avg:152.69ms
step:308/1480 train_time:45499ms step_avg:152.68ms
step:309/1480 train_time:45647ms step_avg:152.67ms
step:310/1480 train_time:45797ms step_avg:152.66ms
step:311/1480 train_time:45945ms step_avg:152.64ms
step:312/1480 train_time:46094ms step_avg:152.63ms
step:313/1480 train_time:46243ms step_avg:152.62ms
step:314/1480 train_time:46392ms step_avg:152.60ms
step:315/1480 train_time:46540ms step_avg:152.59ms
step:316/1480 train_time:46688ms step_avg:152.58ms
step:317/1480 train_time:46837ms step_avg:152.56ms
step:318/1480 train_time:46986ms step_avg:152.55ms
step:319/1480 train_time:47134ms step_avg:152.54ms
step:320/1480 train_time:47283ms step_avg:152.53ms
step:321/1480 train_time:47432ms step_avg:152.52ms
step:322/1480 train_time:47581ms step_avg:152.50ms
step:323/1480 train_time:47730ms step_avg:152.49ms
step:324/1480 train_time:47879ms step_avg:152.48ms
step:325/1480 train_time:48027ms step_avg:152.47ms
step:326/1480 train_time:48176ms step_avg:152.45ms
step:327/1480 train_time:48323ms step_avg:152.44ms
step:328/1480 train_time:48473ms step_avg:152.43ms
step:329/1480 train_time:48621ms step_avg:152.42ms
step:330/1480 train_time:48771ms step_avg:152.41ms
step:331/1480 train_time:48921ms step_avg:152.40ms
step:332/1480 train_time:49073ms step_avg:152.40ms
step:333/1480 train_time:49223ms step_avg:152.39ms
step:334/1480 train_time:49374ms step_avg:152.39ms
step:335/1480 train_time:49524ms step_avg:152.38ms
step:336/1480 train_time:49676ms step_avg:152.38ms
step:337/1480 train_time:49826ms step_avg:152.37ms
step:338/1480 train_time:49978ms step_avg:152.37ms
step:339/1480 train_time:50129ms step_avg:152.37ms
step:340/1480 train_time:50280ms step_avg:152.36ms
step:341/1480 train_time:50430ms step_avg:152.36ms
step:342/1480 train_time:50582ms step_avg:152.36ms
step:343/1480 train_time:50733ms step_avg:152.35ms
step:344/1480 train_time:50883ms step_avg:152.35ms
step:345/1480 train_time:51033ms step_avg:152.34ms
step:346/1480 train_time:51184ms step_avg:152.33ms
step:347/1480 train_time:51335ms step_avg:152.33ms
step:348/1480 train_time:51487ms step_avg:152.33ms
step:349/1480 train_time:51638ms step_avg:152.32ms
step:350/1480 train_time:51791ms step_avg:152.33ms
step:351/1480 train_time:51942ms step_avg:152.32ms
step:352/1480 train_time:52093ms step_avg:152.32ms
step:353/1480 train_time:52243ms step_avg:152.31ms
step:354/1480 train_time:52395ms step_avg:152.31ms
step:355/1480 train_time:52546ms step_avg:152.31ms
step:356/1480 train_time:52697ms step_avg:152.30ms
step:357/1480 train_time:52848ms step_avg:152.30ms
step:358/1480 train_time:53000ms step_avg:152.30ms
step:359/1480 train_time:53151ms step_avg:152.30ms
step:360/1480 train_time:53302ms step_avg:152.29ms
step:361/1480 train_time:53453ms step_avg:152.29ms
step:362/1480 train_time:53603ms step_avg:152.28ms
step:363/1480 train_time:53754ms step_avg:152.28ms
step:364/1480 train_time:53904ms step_avg:152.27ms
step:365/1480 train_time:54055ms step_avg:152.27ms
step:366/1480 train_time:54205ms step_avg:152.26ms
step:367/1480 train_time:54357ms step_avg:152.26ms
step:368/1480 train_time:54506ms step_avg:152.25ms
step:369/1480 train_time:54658ms step_avg:152.25ms
step:370/1480 train_time:54807ms step_avg:152.24ms
step:371/1480 train_time:54959ms step_avg:152.24ms
step:372/1480 train_time:55109ms step_avg:152.24ms
step:373/1480 train_time:55260ms step_avg:152.23ms
step:374/1480 train_time:55413ms step_avg:152.23ms
step:375/1480 train_time:55563ms step_avg:152.23ms
step:375/1480 val_loss:3.8071 train_time:55631ms step_avg:152.41ms
step:376/1480 train_time:55722ms step_avg:152.25ms
step:377/1480 train_time:55871ms step_avg:152.24ms
step:378/1480 train_time:56020ms step_avg:152.23ms
step:379/1480 train_time:56182ms step_avg:152.25ms
step:380/1480 train_time:56320ms step_avg:152.22ms
step:381/1480 train_time:56471ms step_avg:152.21ms
step:382/1480 train_time:56620ms step_avg:152.20ms
step:383/1480 train_time:56773ms step_avg:152.21ms
step:384/1480 train_time:56923ms step_avg:152.20ms
step:385/1480 train_time:57074ms step_avg:152.20ms
step:386/1480 train_time:57223ms step_avg:152.19ms
step:387/1480 train_time:57374ms step_avg:152.19ms
step:388/1480 train_time:57524ms step_avg:152.18ms
step:389/1480 train_time:57675ms step_avg:152.18ms
step:390/1480 train_time:57825ms step_avg:152.17ms
step:391/1480 train_time:57976ms step_avg:152.17ms
step:392/1480 train_time:58127ms step_avg:152.17ms
step:393/1480 train_time:58278ms step_avg:152.16ms
step:394/1480 train_time:58429ms step_avg:152.16ms
step:395/1480 train_time:58579ms step_avg:152.15ms
step:396/1480 train_time:58729ms step_avg:152.15ms
step:397/1480 train_time:58879ms step_avg:152.14ms
step:398/1480 train_time:59029ms step_avg:152.14ms
step:399/1480 train_time:59179ms step_avg:152.13ms
step:400/1480 train_time:59331ms step_avg:152.13ms
step:401/1480 train_time:59481ms step_avg:152.13ms
step:402/1480 train_time:59632ms step_avg:152.12ms
step:403/1480 train_time:59783ms step_avg:152.12ms
step:404/1480 train_time:59935ms step_avg:152.12ms
step:405/1480 train_time:60087ms step_avg:152.12ms
step:406/1480 train_time:60237ms step_avg:152.11ms
step:407/1480 train_time:60389ms step_avg:152.11ms
step:408/1480 train_time:60540ms step_avg:152.11ms
step:409/1480 train_time:60691ms step_avg:152.11ms
step:410/1480 train_time:60841ms step_avg:152.10ms
step:411/1480 train_time:60992ms step_avg:152.10ms
step:412/1480 train_time:61142ms step_avg:152.10ms
step:413/1480 train_time:61294ms step_avg:152.09ms
step:414/1480 train_time:61444ms step_avg:152.09ms
step:415/1480 train_time:61595ms step_avg:152.09ms
step:416/1480 train_time:61745ms step_avg:152.08ms
step:417/1480 train_time:61896ms step_avg:152.08ms
step:418/1480 train_time:62046ms step_avg:152.07ms
step:419/1480 train_time:62197ms step_avg:152.07ms
step:420/1480 train_time:62348ms step_avg:152.07ms
step:421/1480 train_time:62498ms step_avg:152.06ms
step:422/1480 train_time:62648ms step_avg:152.06ms
step:423/1480 train_time:62799ms step_avg:152.05ms
step:424/1480 train_time:62949ms step_avg:152.05ms
step:425/1480 train_time:63100ms step_avg:152.05ms
step:426/1480 train_time:63250ms step_avg:152.04ms
step:427/1480 train_time:63401ms step_avg:152.04ms
step:428/1480 train_time:63551ms step_avg:152.04ms
step:429/1480 train_time:63702ms step_avg:152.03ms
step:430/1480 train_time:63855ms step_avg:152.03ms
step:431/1480 train_time:64006ms step_avg:152.03ms
step:432/1480 train_time:64156ms step_avg:152.03ms
step:433/1480 train_time:64307ms step_avg:152.03ms
step:434/1480 train_time:64458ms step_avg:152.02ms
step:435/1480 train_time:64609ms step_avg:152.02ms
step:436/1480 train_time:64760ms step_avg:152.02ms
step:437/1480 train_time:64912ms step_avg:152.02ms
step:438/1480 train_time:65063ms step_avg:152.02ms
step:439/1480 train_time:65215ms step_avg:152.02ms
step:440/1480 train_time:65366ms step_avg:152.01ms
step:441/1480 train_time:65518ms step_avg:152.01ms
step:442/1480 train_time:65671ms step_avg:152.02ms
step:443/1480 train_time:65822ms step_avg:152.01ms
step:444/1480 train_time:65975ms step_avg:152.02ms
step:445/1480 train_time:66127ms step_avg:152.02ms
step:446/1480 train_time:66280ms step_avg:152.02ms
step:447/1480 train_time:66434ms step_avg:152.02ms
step:448/1480 train_time:66587ms step_avg:152.02ms
step:449/1480 train_time:66740ms step_avg:152.03ms
step:450/1480 train_time:66894ms step_avg:152.03ms
step:451/1480 train_time:67047ms step_avg:152.03ms
step:452/1480 train_time:67199ms step_avg:152.03ms
step:453/1480 train_time:67351ms step_avg:152.03ms
step:454/1480 train_time:67505ms step_avg:152.04ms
step:455/1480 train_time:67658ms step_avg:152.04ms
step:456/1480 train_time:67812ms step_avg:152.04ms
step:457/1480 train_time:67965ms step_avg:152.05ms
step:458/1480 train_time:68118ms step_avg:152.05ms
step:459/1480 train_time:68273ms step_avg:152.05ms
step:460/1480 train_time:68424ms step_avg:152.05ms
step:461/1480 train_time:68576ms step_avg:152.05ms
step:462/1480 train_time:68729ms step_avg:152.06ms
step:463/1480 train_time:68881ms step_avg:152.05ms
step:464/1480 train_time:69034ms step_avg:152.06ms
step:465/1480 train_time:69187ms step_avg:152.06ms
step:466/1480 train_time:69340ms step_avg:152.06ms
step:467/1480 train_time:69494ms step_avg:152.07ms
step:468/1480 train_time:69647ms step_avg:152.07ms
step:469/1480 train_time:69800ms step_avg:152.07ms
step:470/1480 train_time:69952ms step_avg:152.07ms
step:471/1480 train_time:70104ms step_avg:152.07ms
step:472/1480 train_time:70256ms step_avg:152.07ms
step:473/1480 train_time:70410ms step_avg:152.07ms
step:474/1480 train_time:70563ms step_avg:152.08ms
step:475/1480 train_time:70716ms step_avg:152.08ms
step:476/1480 train_time:70870ms step_avg:152.08ms
step:477/1480 train_time:71022ms step_avg:152.08ms
step:478/1480 train_time:71176ms step_avg:152.08ms
step:479/1480 train_time:71328ms step_avg:152.08ms
step:480/1480 train_time:71480ms step_avg:152.09ms
step:481/1480 train_time:71633ms step_avg:152.09ms
step:482/1480 train_time:71787ms step_avg:152.09ms
step:483/1480 train_time:71940ms step_avg:152.09ms
step:484/1480 train_time:72093ms step_avg:152.10ms
step:485/1480 train_time:72246ms step_avg:152.10ms
step:486/1480 train_time:72399ms step_avg:152.10ms
step:487/1480 train_time:72552ms step_avg:152.10ms
step:488/1480 train_time:72705ms step_avg:152.10ms
step:489/1480 train_time:72858ms step_avg:152.10ms
step:490/1480 train_time:73012ms step_avg:152.11ms
step:491/1480 train_time:73165ms step_avg:152.11ms
step:492/1480 train_time:73317ms step_avg:152.11ms
step:493/1480 train_time:73471ms step_avg:152.11ms
step:494/1480 train_time:73622ms step_avg:152.11ms
step:495/1480 train_time:73775ms step_avg:152.11ms
step:496/1480 train_time:73928ms step_avg:152.12ms
step:497/1480 train_time:74081ms step_avg:152.12ms
step:498/1480 train_time:74235ms step_avg:152.12ms
step:499/1480 train_time:74388ms step_avg:152.12ms
step:500/1480 train_time:74542ms step_avg:152.13ms
step:500/1480 val_loss:3.6846 train_time:74610ms step_avg:152.27ms
step:501/1480 train_time:74709ms step_avg:152.16ms
step:502/1480 train_time:74855ms step_avg:152.14ms
step:503/1480 train_time:75008ms step_avg:152.15ms
step:504/1480 train_time:75161ms step_avg:152.15ms
step:505/1480 train_time:75313ms step_avg:152.15ms
step:506/1480 train_time:75465ms step_avg:152.15ms
step:507/1480 train_time:75618ms step_avg:152.15ms
step:508/1480 train_time:75771ms step_avg:152.15ms
step:509/1480 train_time:75925ms step_avg:152.15ms
step:510/1480 train_time:76079ms step_avg:152.16ms
step:511/1480 train_time:76232ms step_avg:152.16ms
step:512/1480 train_time:76385ms step_avg:152.16ms
step:513/1480 train_time:76538ms step_avg:152.16ms
step:514/1480 train_time:76690ms step_avg:152.16ms
step:515/1480 train_time:76844ms step_avg:152.17ms
step:516/1480 train_time:76998ms step_avg:152.17ms
step:517/1480 train_time:77153ms step_avg:152.18ms
step:518/1480 train_time:77306ms step_avg:152.18ms
step:519/1480 train_time:77458ms step_avg:152.18ms
step:520/1480 train_time:77611ms step_avg:152.18ms
step:521/1480 train_time:77764ms step_avg:152.18ms
step:522/1480 train_time:77919ms step_avg:152.18ms
step:523/1480 train_time:78071ms step_avg:152.19ms
step:524/1480 train_time:78225ms step_avg:152.19ms
step:525/1480 train_time:78378ms step_avg:152.19ms
step:526/1480 train_time:78530ms step_avg:152.19ms
step:527/1480 train_time:78683ms step_avg:152.19ms
step:528/1480 train_time:78835ms step_avg:152.19ms
step:529/1480 train_time:78987ms step_avg:152.19ms
step:530/1480 train_time:79141ms step_avg:152.19ms
step:531/1480 train_time:79295ms step_avg:152.20ms
step:532/1480 train_time:79448ms step_avg:152.20ms
step:533/1480 train_time:79601ms step_avg:152.20ms
step:534/1480 train_time:79753ms step_avg:152.20ms
step:535/1480 train_time:79905ms step_avg:152.20ms
step:536/1480 train_time:80057ms step_avg:152.20ms
step:537/1480 train_time:80210ms step_avg:152.20ms
step:538/1480 train_time:80364ms step_avg:152.20ms
step:539/1480 train_time:80519ms step_avg:152.21ms
step:540/1480 train_time:80674ms step_avg:152.21ms
step:541/1480 train_time:80826ms step_avg:152.22ms
step:542/1480 train_time:80979ms step_avg:152.22ms
step:543/1480 train_time:81131ms step_avg:152.22ms
step:544/1480 train_time:81284ms step_avg:152.22ms
step:545/1480 train_time:81436ms step_avg:152.22ms
step:546/1480 train_time:81590ms step_avg:152.22ms
step:547/1480 train_time:81743ms step_avg:152.22ms
step:548/1480 train_time:81895ms step_avg:152.22ms
step:549/1480 train_time:82048ms step_avg:152.22ms
step:550/1480 train_time:82203ms step_avg:152.23ms
step:551/1480 train_time:82358ms step_avg:152.23ms
step:552/1480 train_time:82513ms step_avg:152.24ms
step:553/1480 train_time:82669ms step_avg:152.25ms
step:554/1480 train_time:82824ms step_avg:152.25ms
step:555/1480 train_time:82979ms step_avg:152.26ms
step:556/1480 train_time:83133ms step_avg:152.26ms
step:557/1480 train_time:83289ms step_avg:152.27ms
step:558/1480 train_time:83444ms step_avg:152.27ms
step:559/1480 train_time:83600ms step_avg:152.28ms
step:560/1480 train_time:83755ms step_avg:152.28ms
step:561/1480 train_time:83909ms step_avg:152.28ms
step:562/1480 train_time:84063ms step_avg:152.29ms
step:563/1480 train_time:84218ms step_avg:152.29ms
step:564/1480 train_time:84373ms step_avg:152.30ms
step:565/1480 train_time:84527ms step_avg:152.30ms
step:566/1480 train_time:84682ms step_avg:152.31ms
step:567/1480 train_time:84837ms step_avg:152.31ms
step:568/1480 train_time:84991ms step_avg:152.31ms
step:569/1480 train_time:85156ms step_avg:152.34ms
step:570/1480 train_time:85302ms step_avg:152.32ms
step:571/1480 train_time:85455ms step_avg:152.33ms
step:572/1480 train_time:85610ms step_avg:152.33ms
step:573/1480 train_time:85764ms step_avg:152.33ms
step:574/1480 train_time:85921ms step_avg:152.34ms
step:575/1480 train_time:86076ms step_avg:152.35ms
step:576/1480 train_time:86231ms step_avg:152.35ms
step:577/1480 train_time:86386ms step_avg:152.36ms
step:578/1480 train_time:86541ms step_avg:152.36ms
step:579/1480 train_time:86695ms step_avg:152.36ms
step:580/1480 train_time:86848ms step_avg:152.37ms
step:581/1480 train_time:87004ms step_avg:152.37ms
step:582/1480 train_time:87158ms step_avg:152.37ms
step:583/1480 train_time:87312ms step_avg:152.38ms
step:584/1480 train_time:87465ms step_avg:152.38ms
step:585/1480 train_time:87621ms step_avg:152.38ms
step:586/1480 train_time:87775ms step_avg:152.39ms
step:587/1480 train_time:87931ms step_avg:152.39ms
step:588/1480 train_time:88085ms step_avg:152.40ms
step:589/1480 train_time:88241ms step_avg:152.40ms
step:590/1480 train_time:88395ms step_avg:152.41ms
step:591/1480 train_time:88550ms step_avg:152.41ms
step:592/1480 train_time:88705ms step_avg:152.41ms
step:593/1480 train_time:88860ms step_avg:152.42ms
step:594/1480 train_time:89014ms step_avg:152.42ms
step:595/1480 train_time:89171ms step_avg:152.43ms
step:596/1480 train_time:89326ms step_avg:152.43ms
step:597/1480 train_time:89481ms step_avg:152.44ms
step:598/1480 train_time:89635ms step_avg:152.44ms
step:599/1480 train_time:89790ms step_avg:152.45ms
step:600/1480 train_time:89945ms step_avg:152.45ms
step:601/1480 train_time:90102ms step_avg:152.46ms
step:602/1480 train_time:90257ms step_avg:152.46ms
step:603/1480 train_time:90411ms step_avg:152.46ms
step:604/1480 train_time:90566ms step_avg:152.47ms
step:605/1480 train_time:90722ms step_avg:152.47ms
step:606/1480 train_time:90878ms step_avg:152.48ms
step:607/1480 train_time:91034ms step_avg:152.49ms
step:608/1480 train_time:91188ms step_avg:152.49ms
step:609/1480 train_time:91343ms step_avg:152.49ms
step:610/1480 train_time:91496ms step_avg:152.49ms
step:611/1480 train_time:91651ms step_avg:152.50ms
step:612/1480 train_time:91806ms step_avg:152.50ms
step:613/1480 train_time:91960ms step_avg:152.50ms
step:614/1480 train_time:92115ms step_avg:152.51ms
step:615/1480 train_time:92269ms step_avg:152.51ms
step:616/1480 train_time:92424ms step_avg:152.51ms
step:617/1480 train_time:92580ms step_avg:152.52ms
step:618/1480 train_time:92734ms step_avg:152.52ms
step:619/1480 train_time:92889ms step_avg:152.53ms
step:620/1480 train_time:93045ms step_avg:152.53ms
step:621/1480 train_time:93201ms step_avg:152.54ms
step:622/1480 train_time:93355ms step_avg:152.54ms
step:623/1480 train_time:93511ms step_avg:152.55ms
step:624/1480 train_time:93665ms step_avg:152.55ms
step:625/1480 train_time:93821ms step_avg:152.55ms
step:625/1480 val_loss:3.6019 train_time:93892ms step_avg:152.67ms
step:626/1480 train_time:93985ms step_avg:152.57ms
step:627/1480 train_time:94136ms step_avg:152.57ms
step:628/1480 train_time:94290ms step_avg:152.57ms
step:629/1480 train_time:94445ms step_avg:152.58ms
step:630/1480 train_time:94599ms step_avg:152.58ms
step:631/1480 train_time:94753ms step_avg:152.58ms
step:632/1480 train_time:94907ms step_avg:152.58ms
step:633/1480 train_time:95061ms step_avg:152.59ms
step:634/1480 train_time:95217ms step_avg:152.59ms
step:635/1480 train_time:95373ms step_avg:152.60ms
step:636/1480 train_time:95527ms step_avg:152.60ms
step:637/1480 train_time:95683ms step_avg:152.60ms
step:638/1480 train_time:95837ms step_avg:152.61ms
step:639/1480 train_time:95991ms step_avg:152.61ms
step:640/1480 train_time:96145ms step_avg:152.61ms
step:641/1480 train_time:96300ms step_avg:152.61ms
step:642/1480 train_time:96454ms step_avg:152.62ms
step:643/1480 train_time:96609ms step_avg:152.62ms
step:644/1480 train_time:96763ms step_avg:152.62ms
step:645/1480 train_time:96919ms step_avg:152.63ms
step:646/1480 train_time:97073ms step_avg:152.63ms
step:647/1480 train_time:97228ms step_avg:152.63ms
step:648/1480 train_time:97384ms step_avg:152.64ms
step:649/1480 train_time:97539ms step_avg:152.64ms
step:650/1480 train_time:97694ms step_avg:152.65ms
step:651/1480 train_time:97848ms step_avg:152.65ms
step:652/1480 train_time:98004ms step_avg:152.65ms
step:653/1480 train_time:98158ms step_avg:152.66ms
step:654/1480 train_time:98313ms step_avg:152.66ms
step:655/1480 train_time:98467ms step_avg:152.66ms
step:656/1480 train_time:98621ms step_avg:152.66ms
step:657/1480 train_time:98775ms step_avg:152.67ms
step:658/1480 train_time:98930ms step_avg:152.67ms
step:659/1480 train_time:99086ms step_avg:152.68ms
step:660/1480 train_time:99243ms step_avg:152.68ms
step:661/1480 train_time:99400ms step_avg:152.69ms
step:662/1480 train_time:99557ms step_avg:152.70ms
step:663/1480 train_time:99713ms step_avg:152.70ms
step:664/1480 train_time:99870ms step_avg:152.71ms
step:665/1480 train_time:100026ms step_avg:152.71ms
step:666/1480 train_time:100182ms step_avg:152.72ms
step:667/1480 train_time:100338ms step_avg:152.72ms
step:668/1480 train_time:100494ms step_avg:152.73ms
step:669/1480 train_time:100650ms step_avg:152.73ms
step:670/1480 train_time:100806ms step_avg:152.74ms
step:671/1480 train_time:100962ms step_avg:152.74ms
step:672/1480 train_time:101121ms step_avg:152.75ms
step:673/1480 train_time:101277ms step_avg:152.76ms
step:674/1480 train_time:101434ms step_avg:152.76ms
step:675/1480 train_time:101591ms step_avg:152.77ms
step:676/1480 train_time:101748ms step_avg:152.77ms
step:677/1480 train_time:101904ms step_avg:152.78ms
step:678/1480 train_time:102059ms step_avg:152.78ms
step:679/1480 train_time:102216ms step_avg:152.79ms
step:680/1480 train_time:102374ms step_avg:152.80ms
step:681/1480 train_time:102529ms step_avg:152.80ms
step:682/1480 train_time:102686ms step_avg:152.81ms
step:683/1480 train_time:102842ms step_avg:152.81ms
step:684/1480 train_time:102998ms step_avg:152.82ms
step:685/1480 train_time:103154ms step_avg:152.82ms
step:686/1480 train_time:103311ms step_avg:152.83ms
step:687/1480 train_time:103467ms step_avg:152.83ms
step:688/1480 train_time:103624ms step_avg:152.84ms
step:689/1480 train_time:103781ms step_avg:152.84ms
step:690/1480 train_time:103940ms step_avg:152.85ms
step:691/1480 train_time:104097ms step_avg:152.86ms
step:692/1480 train_time:104253ms step_avg:152.86ms
step:693/1480 train_time:104410ms step_avg:152.87ms
step:694/1480 train_time:104566ms step_avg:152.87ms
step:695/1480 train_time:104721ms step_avg:152.88ms
step:696/1480 train_time:104878ms step_avg:152.88ms
step:697/1480 train_time:105033ms step_avg:152.89ms
step:698/1480 train_time:105189ms step_avg:152.89ms
step:699/1480 train_time:105345ms step_avg:152.89ms
step:700/1480 train_time:105502ms step_avg:152.90ms
step:701/1480 train_time:105657ms step_avg:152.90ms
step:702/1480 train_time:105814ms step_avg:152.91ms
step:703/1480 train_time:105971ms step_avg:152.92ms
step:704/1480 train_time:106126ms step_avg:152.92ms
step:705/1480 train_time:106283ms step_avg:152.93ms
step:706/1480 train_time:106442ms step_avg:152.93ms
step:707/1480 train_time:106599ms step_avg:152.94ms
step:708/1480 train_time:106755ms step_avg:152.94ms
step:709/1480 train_time:106910ms step_avg:152.95ms
step:710/1480 train_time:107065ms step_avg:152.95ms
step:711/1480 train_time:107222ms step_avg:152.96ms
step:712/1480 train_time:107381ms step_avg:152.96ms
step:713/1480 train_time:107537ms step_avg:152.97ms
step:714/1480 train_time:107695ms step_avg:152.98ms
step:715/1480 train_time:107851ms step_avg:152.98ms
step:716/1480 train_time:108006ms step_avg:152.98ms
step:717/1480 train_time:108162ms step_avg:152.99ms
step:718/1480 train_time:108318ms step_avg:152.99ms
step:719/1480 train_time:108472ms step_avg:152.99ms
step:720/1480 train_time:108630ms step_avg:153.00ms
step:721/1480 train_time:108786ms step_avg:153.00ms
step:722/1480 train_time:108943ms step_avg:153.01ms
step:723/1480 train_time:109098ms step_avg:153.01ms
step:724/1480 train_time:109254ms step_avg:153.02ms
step:725/1480 train_time:109410ms step_avg:153.02ms
step:726/1480 train_time:109566ms step_avg:153.03ms
step:727/1480 train_time:109724ms step_avg:153.03ms
step:728/1480 train_time:109881ms step_avg:153.04ms
step:729/1480 train_time:110036ms step_avg:153.04ms
step:730/1480 train_time:110194ms step_avg:153.05ms
step:731/1480 train_time:110350ms step_avg:153.05ms
step:732/1480 train_time:110506ms step_avg:153.05ms
step:733/1480 train_time:110661ms step_avg:153.06ms
step:734/1480 train_time:110819ms step_avg:153.06ms
step:735/1480 train_time:110977ms step_avg:153.07ms
step:736/1480 train_time:111132ms step_avg:153.07ms
step:737/1480 train_time:111288ms step_avg:153.08ms
step:738/1480 train_time:111444ms step_avg:153.08ms
step:739/1480 train_time:111600ms step_avg:153.09ms
step:740/1480 train_time:111759ms step_avg:153.09ms
step:741/1480 train_time:111917ms step_avg:153.10ms
step:742/1480 train_time:112073ms step_avg:153.10ms
step:743/1480 train_time:112228ms step_avg:153.11ms
step:744/1480 train_time:112384ms step_avg:153.11ms
step:745/1480 train_time:112542ms step_avg:153.12ms
step:746/1480 train_time:112697ms step_avg:153.12ms
step:747/1480 train_time:112855ms step_avg:153.13ms
step:748/1480 train_time:113015ms step_avg:153.14ms
step:749/1480 train_time:113171ms step_avg:153.14ms
step:750/1480 train_time:113326ms step_avg:153.14ms
step:750/1480 val_loss:3.5453 train_time:113397ms step_avg:153.24ms
step:751/1480 train_time:113491ms step_avg:153.16ms
step:752/1480 train_time:113645ms step_avg:153.16ms
step:753/1480 train_time:113802ms step_avg:153.17ms
step:754/1480 train_time:113957ms step_avg:153.17ms
step:755/1480 train_time:114113ms step_avg:153.17ms
step:756/1480 train_time:114270ms step_avg:153.18ms
step:757/1480 train_time:114428ms step_avg:153.18ms
step:758/1480 train_time:114585ms step_avg:153.19ms
step:759/1480 train_time:114751ms step_avg:153.21ms
step:760/1480 train_time:114899ms step_avg:153.20ms
step:761/1480 train_time:115054ms step_avg:153.20ms
step:762/1480 train_time:115210ms step_avg:153.21ms
step:763/1480 train_time:115368ms step_avg:153.21ms
step:764/1480 train_time:115524ms step_avg:153.22ms
step:765/1480 train_time:115681ms step_avg:153.22ms
step:766/1480 train_time:115839ms step_avg:153.23ms
step:767/1480 train_time:115995ms step_avg:153.23ms
step:768/1480 train_time:116151ms step_avg:153.23ms
step:769/1480 train_time:116309ms step_avg:153.24ms
step:770/1480 train_time:116467ms step_avg:153.25ms
step:771/1480 train_time:116624ms step_avg:153.25ms
step:772/1480 train_time:116782ms step_avg:153.26ms
step:773/1480 train_time:116940ms step_avg:153.26ms
step:774/1480 train_time:117096ms step_avg:153.27ms
step:775/1480 train_time:117255ms step_avg:153.27ms
step:776/1480 train_time:117413ms step_avg:153.28ms
step:777/1480 train_time:117573ms step_avg:153.29ms
step:778/1480 train_time:117730ms step_avg:153.29ms
step:779/1480 train_time:117888ms step_avg:153.30ms
step:780/1480 train_time:118048ms step_avg:153.31ms
step:781/1480 train_time:118207ms step_avg:153.32ms
step:782/1480 train_time:118366ms step_avg:153.32ms
step:783/1480 train_time:118523ms step_avg:153.33ms
step:784/1480 train_time:118680ms step_avg:153.33ms
step:785/1480 train_time:118838ms step_avg:153.34ms
step:786/1480 train_time:118996ms step_avg:153.35ms
step:787/1480 train_time:119153ms step_avg:153.35ms
step:788/1480 train_time:119314ms step_avg:153.36ms
step:789/1480 train_time:119471ms step_avg:153.36ms
step:790/1480 train_time:119629ms step_avg:153.37ms
step:791/1480 train_time:119790ms step_avg:153.38ms
step:792/1480 train_time:119947ms step_avg:153.39ms
step:793/1480 train_time:120105ms step_avg:153.39ms
step:794/1480 train_time:120263ms step_avg:153.40ms
step:795/1480 train_time:120424ms step_avg:153.41ms
step:796/1480 train_time:120585ms step_avg:153.42ms
step:797/1480 train_time:120745ms step_avg:153.42ms
step:798/1480 train_time:120904ms step_avg:153.43ms
step:799/1480 train_time:121065ms step_avg:153.44ms
step:800/1480 train_time:121223ms step_avg:153.45ms
step:801/1480 train_time:121381ms step_avg:153.45ms
step:802/1480 train_time:121540ms step_avg:153.46ms
step:803/1480 train_time:121697ms step_avg:153.46ms
step:804/1480 train_time:121854ms step_avg:153.47ms
step:805/1480 train_time:122013ms step_avg:153.48ms
step:806/1480 train_time:122171ms step_avg:153.48ms
step:807/1480 train_time:122328ms step_avg:153.49ms
step:808/1480 train_time:122487ms step_avg:153.49ms
step:809/1480 train_time:122645ms step_avg:153.50ms
step:810/1480 train_time:122802ms step_avg:153.50ms
step:811/1480 train_time:122958ms step_avg:153.51ms
step:812/1480 train_time:123115ms step_avg:153.51ms
step:813/1480 train_time:123271ms step_avg:153.51ms
step:814/1480 train_time:123429ms step_avg:153.52ms
step:815/1480 train_time:123586ms step_avg:153.52ms
step:816/1480 train_time:123747ms step_avg:153.53ms
step:817/1480 train_time:123905ms step_avg:153.54ms
step:818/1480 train_time:124063ms step_avg:153.54ms
step:819/1480 train_time:124221ms step_avg:153.55ms
step:820/1480 train_time:124380ms step_avg:153.56ms
step:821/1480 train_time:124538ms step_avg:153.56ms
step:822/1480 train_time:124694ms step_avg:153.56ms
step:823/1480 train_time:124851ms step_avg:153.57ms
step:824/1480 train_time:125009ms step_avg:153.57ms
step:825/1480 train_time:125169ms step_avg:153.58ms
step:826/1480 train_time:125329ms step_avg:153.59ms
step:827/1480 train_time:125488ms step_avg:153.60ms
step:828/1480 train_time:125646ms step_avg:153.60ms
step:829/1480 train_time:125805ms step_avg:153.61ms
step:830/1480 train_time:125965ms step_avg:153.62ms
step:831/1480 train_time:126124ms step_avg:153.62ms
step:832/1480 train_time:126284ms step_avg:153.63ms
step:833/1480 train_time:126442ms step_avg:153.64ms
step:834/1480 train_time:126599ms step_avg:153.64ms
step:835/1480 train_time:126755ms step_avg:153.64ms
step:836/1480 train_time:126913ms step_avg:153.65ms
step:837/1480 train_time:127071ms step_avg:153.65ms
step:838/1480 train_time:127229ms step_avg:153.66ms
step:839/1480 train_time:127387ms step_avg:153.66ms
step:840/1480 train_time:127545ms step_avg:153.67ms
step:841/1480 train_time:127701ms step_avg:153.67ms
step:842/1480 train_time:127859ms step_avg:153.68ms
step:843/1480 train_time:128015ms step_avg:153.68ms
step:844/1480 train_time:128171ms step_avg:153.68ms
step:845/1480 train_time:128329ms step_avg:153.69ms
step:846/1480 train_time:128488ms step_avg:153.69ms
step:847/1480 train_time:128648ms step_avg:153.70ms
step:848/1480 train_time:128806ms step_avg:153.71ms
step:849/1480 train_time:128963ms step_avg:153.71ms
step:850/1480 train_time:129121ms step_avg:153.72ms
step:851/1480 train_time:129282ms step_avg:153.72ms
step:852/1480 train_time:129440ms step_avg:153.73ms
step:853/1480 train_time:129598ms step_avg:153.73ms
step:854/1480 train_time:129755ms step_avg:153.74ms
step:855/1480 train_time:129913ms step_avg:153.74ms
step:856/1480 train_time:130070ms step_avg:153.75ms
step:857/1480 train_time:130228ms step_avg:153.75ms
step:858/1480 train_time:130389ms step_avg:153.76ms
step:859/1480 train_time:130548ms step_avg:153.77ms
step:860/1480 train_time:130706ms step_avg:153.77ms
step:861/1480 train_time:130866ms step_avg:153.78ms
step:862/1480 train_time:131026ms step_avg:153.79ms
step:863/1480 train_time:131186ms step_avg:153.79ms
step:864/1480 train_time:131346ms step_avg:153.80ms
step:865/1480 train_time:131503ms step_avg:153.80ms
step:866/1480 train_time:131661ms step_avg:153.81ms
step:867/1480 train_time:131820ms step_avg:153.82ms
step:868/1480 train_time:131976ms step_avg:153.82ms
step:869/1480 train_time:132133ms step_avg:153.82ms
step:870/1480 train_time:132291ms step_avg:153.83ms
step:871/1480 train_time:132447ms step_avg:153.83ms
step:872/1480 train_time:132608ms step_avg:153.84ms
step:873/1480 train_time:132764ms step_avg:153.84ms
step:874/1480 train_time:132924ms step_avg:153.85ms
step:875/1480 train_time:133084ms step_avg:153.85ms
step:875/1480 val_loss:3.5001 train_time:133157ms step_avg:153.94ms
step:876/1480 train_time:133248ms step_avg:153.87ms
step:877/1480 train_time:133405ms step_avg:153.87ms
step:878/1480 train_time:133562ms step_avg:153.87ms
step:879/1480 train_time:133720ms step_avg:153.88ms
step:880/1480 train_time:133877ms step_avg:153.88ms
step:881/1480 train_time:134035ms step_avg:153.89ms
step:882/1480 train_time:134194ms step_avg:153.89ms
step:883/1480 train_time:134354ms step_avg:153.90ms
step:884/1480 train_time:134514ms step_avg:153.91ms
step:885/1480 train_time:134674ms step_avg:153.91ms
step:886/1480 train_time:134834ms step_avg:153.92ms
step:887/1480 train_time:134994ms step_avg:153.93ms
step:888/1480 train_time:135156ms step_avg:153.94ms
step:889/1480 train_time:135316ms step_avg:153.94ms
step:890/1480 train_time:135474ms step_avg:153.95ms
step:891/1480 train_time:135632ms step_avg:153.95ms
step:892/1480 train_time:135792ms step_avg:153.96ms
step:893/1480 train_time:135951ms step_avg:153.96ms
step:894/1480 train_time:136111ms step_avg:153.97ms
step:895/1480 train_time:136272ms step_avg:153.98ms
step:896/1480 train_time:136430ms step_avg:153.98ms
step:897/1480 train_time:136592ms step_avg:153.99ms
step:898/1480 train_time:136752ms step_avg:154.00ms
step:899/1480 train_time:136911ms step_avg:154.01ms
step:900/1480 train_time:137069ms step_avg:154.01ms
step:901/1480 train_time:137228ms step_avg:154.02ms
step:902/1480 train_time:137386ms step_avg:154.02ms
step:903/1480 train_time:137547ms step_avg:154.03ms
step:904/1480 train_time:137709ms step_avg:154.04ms
step:905/1480 train_time:137867ms step_avg:154.04ms
step:906/1480 train_time:138028ms step_avg:154.05ms
step:907/1480 train_time:138190ms step_avg:154.06ms
step:908/1480 train_time:138349ms step_avg:154.06ms
step:909/1480 train_time:138508ms step_avg:154.07ms
step:910/1480 train_time:138674ms step_avg:154.08ms
step:911/1480 train_time:138832ms step_avg:154.09ms
step:912/1480 train_time:138993ms step_avg:154.09ms
step:913/1480 train_time:139155ms step_avg:154.10ms
step:914/1480 train_time:139315ms step_avg:154.11ms
step:915/1480 train_time:139477ms step_avg:154.12ms
step:916/1480 train_time:139635ms step_avg:154.12ms
step:917/1480 train_time:139794ms step_avg:154.13ms
step:918/1480 train_time:139955ms step_avg:154.14ms
step:919/1480 train_time:140115ms step_avg:154.14ms
step:920/1480 train_time:140274ms step_avg:154.15ms
step:921/1480 train_time:140433ms step_avg:154.15ms
step:922/1480 train_time:140594ms step_avg:154.16ms
step:923/1480 train_time:140752ms step_avg:154.16ms
step:924/1480 train_time:140911ms step_avg:154.17ms
step:925/1480 train_time:141071ms step_avg:154.18ms
step:926/1480 train_time:141231ms step_avg:154.18ms
step:927/1480 train_time:141390ms step_avg:154.19ms
step:928/1480 train_time:141550ms step_avg:154.19ms
step:929/1480 train_time:141709ms step_avg:154.20ms
step:930/1480 train_time:141869ms step_avg:154.20ms
step:931/1480 train_time:142028ms step_avg:154.21ms
step:932/1480 train_time:142189ms step_avg:154.22ms
step:933/1480 train_time:142348ms step_avg:154.22ms
step:934/1480 train_time:142508ms step_avg:154.23ms
step:935/1480 train_time:142670ms step_avg:154.24ms
step:936/1480 train_time:142828ms step_avg:154.24ms
step:937/1480 train_time:142991ms step_avg:154.25ms
step:938/1480 train_time:143150ms step_avg:154.26ms
step:939/1480 train_time:143312ms step_avg:154.27ms
step:940/1480 train_time:143473ms step_avg:154.27ms
step:941/1480 train_time:143631ms step_avg:154.28ms
step:942/1480 train_time:143790ms step_avg:154.28ms
step:943/1480 train_time:143950ms step_avg:154.29ms
step:944/1480 train_time:144113ms step_avg:154.30ms
step:945/1480 train_time:144272ms step_avg:154.30ms
step:946/1480 train_time:144434ms step_avg:154.31ms
step:947/1480 train_time:144594ms step_avg:154.32ms
step:948/1480 train_time:144753ms step_avg:154.32ms
step:949/1480 train_time:144920ms step_avg:154.33ms
step:950/1480 train_time:145072ms step_avg:154.33ms
step:951/1480 train_time:145234ms step_avg:154.34ms
step:952/1480 train_time:145393ms step_avg:154.35ms
step:953/1480 train_time:145556ms step_avg:154.35ms
step:954/1480 train_time:145717ms step_avg:154.36ms
step:955/1480 train_time:145874ms step_avg:154.36ms
step:956/1480 train_time:146032ms step_avg:154.37ms
step:957/1480 train_time:146194ms step_avg:154.38ms
step:958/1480 train_time:146357ms step_avg:154.38ms
step:959/1480 train_time:146514ms step_avg:154.39ms
step:960/1480 train_time:146674ms step_avg:154.39ms
step:961/1480 train_time:146833ms step_avg:154.40ms
step:962/1480 train_time:146992ms step_avg:154.40ms
step:963/1480 train_time:147153ms step_avg:154.41ms
step:964/1480 train_time:147314ms step_avg:154.42ms
step:965/1480 train_time:147473ms step_avg:154.42ms
step:966/1480 train_time:147631ms step_avg:154.43ms
step:967/1480 train_time:147790ms step_avg:154.43ms
step:968/1480 train_time:147951ms step_avg:154.44ms
step:969/1480 train_time:148111ms step_avg:154.44ms
step:970/1480 train_time:148270ms step_avg:154.45ms
step:971/1480 train_time:148428ms step_avg:154.45ms
step:972/1480 train_time:148587ms step_avg:154.46ms
step:973/1480 train_time:148746ms step_avg:154.46ms
step:974/1480 train_time:148906ms step_avg:154.47ms
step:975/1480 train_time:149067ms step_avg:154.47ms
step:976/1480 train_time:149228ms step_avg:154.48ms
step:977/1480 train_time:149387ms step_avg:154.49ms
step:978/1480 train_time:149548ms step_avg:154.49ms
step:979/1480 train_time:149708ms step_avg:154.50ms
step:980/1480 train_time:149868ms step_avg:154.50ms
step:981/1480 train_time:150030ms step_avg:154.51ms
step:982/1480 train_time:150190ms step_avg:154.52ms
step:983/1480 train_time:150350ms step_avg:154.52ms
step:984/1480 train_time:150508ms step_avg:154.53ms
step:985/1480 train_time:150671ms step_avg:154.53ms
step:986/1480 train_time:150830ms step_avg:154.54ms
step:987/1480 train_time:150988ms step_avg:154.54ms
step:988/1480 train_time:151147ms step_avg:154.55ms
step:989/1480 train_time:151305ms step_avg:154.55ms
step:990/1480 train_time:151470ms step_avg:154.56ms
step:991/1480 train_time:151631ms step_avg:154.57ms
step:992/1480 train_time:151796ms step_avg:154.58ms
step:993/1480 train_time:151964ms step_avg:154.59ms
step:994/1480 train_time:152124ms step_avg:154.60ms
step:995/1480 train_time:152284ms step_avg:154.60ms
step:996/1480 train_time:152441ms step_avg:154.61ms
step:997/1480 train_time:152601ms step_avg:154.61ms
step:998/1480 train_time:152762ms step_avg:154.62ms
step:999/1480 train_time:152923ms step_avg:154.62ms
step:1000/1480 train_time:153082ms step_avg:154.63ms
step:1000/1480 val_loss:3.4370 train_time:153155ms step_avg:154.70ms
step:1001/1480 train_time:153246ms step_avg:154.64ms
step:1002/1480 train_time:153402ms step_avg:154.64ms
step:1003/1480 train_time:153566ms step_avg:154.65ms
step:1004/1480 train_time:153727ms step_avg:154.66ms
step:1005/1480 train_time:153887ms step_avg:154.66ms
step:1006/1480 train_time:154048ms step_avg:154.67ms
step:1007/1480 train_time:154209ms step_avg:154.67ms
step:1008/1480 train_time:154370ms step_avg:154.68ms
step:1009/1480 train_time:154535ms step_avg:154.69ms
step:1010/1480 train_time:154694ms step_avg:154.69ms
step:1011/1480 train_time:154855ms step_avg:154.70ms
step:1012/1480 train_time:155015ms step_avg:154.71ms
step:1013/1480 train_time:155177ms step_avg:154.71ms
step:1014/1480 train_time:155338ms step_avg:154.72ms
step:1015/1480 train_time:155500ms step_avg:154.73ms
step:1016/1480 train_time:155659ms step_avg:154.73ms
step:1017/1480 train_time:155822ms step_avg:154.74ms
step:1018/1480 train_time:155983ms step_avg:154.74ms
step:1019/1480 train_time:156144ms step_avg:154.75ms
step:1020/1480 train_time:156305ms step_avg:154.76ms
step:1021/1480 train_time:156466ms step_avg:154.76ms
step:1022/1480 train_time:156626ms step_avg:154.77ms
step:1023/1480 train_time:156789ms step_avg:154.78ms
step:1024/1480 train_time:156949ms step_avg:154.78ms
step:1025/1480 train_time:157112ms step_avg:154.79ms
step:1026/1480 train_time:157272ms step_avg:154.80ms
step:1027/1480 train_time:157432ms step_avg:154.80ms
step:1028/1480 train_time:157594ms step_avg:154.81ms
step:1029/1480 train_time:157758ms step_avg:154.82ms
step:1030/1480 train_time:157917ms step_avg:154.82ms
step:1031/1480 train_time:158076ms step_avg:154.82ms
step:1032/1480 train_time:158240ms step_avg:154.83ms
step:1033/1480 train_time:158398ms step_avg:154.84ms
step:1034/1480 train_time:158559ms step_avg:154.84ms
step:1035/1480 train_time:158719ms step_avg:154.85ms
step:1036/1480 train_time:158878ms step_avg:154.85ms
step:1037/1480 train_time:159039ms step_avg:154.86ms
step:1038/1480 train_time:159197ms step_avg:154.86ms
step:1039/1480 train_time:159360ms step_avg:154.87ms
step:1040/1480 train_time:159519ms step_avg:154.87ms
step:1041/1480 train_time:159679ms step_avg:154.88ms
step:1042/1480 train_time:159837ms step_avg:154.88ms
step:1043/1480 train_time:159994ms step_avg:154.88ms
step:1044/1480 train_time:160155ms step_avg:154.89ms
step:1045/1480 train_time:160317ms step_avg:154.90ms
step:1046/1480 train_time:160476ms step_avg:154.90ms
step:1047/1480 train_time:160637ms step_avg:154.91ms
step:1048/1480 train_time:160798ms step_avg:154.91ms
step:1049/1480 train_time:160958ms step_avg:154.92ms
step:1050/1480 train_time:161120ms step_avg:154.92ms
step:1051/1480 train_time:161282ms step_avg:154.93ms
step:1052/1480 train_time:161442ms step_avg:154.94ms
step:1053/1480 train_time:161601ms step_avg:154.94ms
step:1054/1480 train_time:161760ms step_avg:154.94ms
step:1055/1480 train_time:161920ms step_avg:154.95ms
step:1056/1480 train_time:162079ms step_avg:154.95ms
step:1057/1480 train_time:162239ms step_avg:154.96ms
step:1058/1480 train_time:162400ms step_avg:154.96ms
step:1059/1480 train_time:162562ms step_avg:154.97ms
step:1060/1480 train_time:162723ms step_avg:154.97ms
step:1061/1480 train_time:162881ms step_avg:154.98ms
step:1062/1480 train_time:163041ms step_avg:154.98ms
step:1063/1480 train_time:163199ms step_avg:154.99ms
step:1064/1480 train_time:163357ms step_avg:154.99ms
step:1065/1480 train_time:163517ms step_avg:154.99ms
step:1066/1480 train_time:163679ms step_avg:155.00ms
step:1067/1480 train_time:163841ms step_avg:155.01ms
step:1068/1480 train_time:164001ms step_avg:155.01ms
step:1069/1480 train_time:164165ms step_avg:155.02ms
step:1070/1480 train_time:164324ms step_avg:155.02ms
step:1071/1480 train_time:164490ms step_avg:155.03ms
step:1072/1480 train_time:164650ms step_avg:155.04ms
step:1073/1480 train_time:164810ms step_avg:155.04ms
step:1074/1480 train_time:164970ms step_avg:155.05ms
step:1075/1480 train_time:165132ms step_avg:155.05ms
step:1076/1480 train_time:165292ms step_avg:155.06ms
step:1077/1480 train_time:165451ms step_avg:155.06ms
step:1078/1480 train_time:165618ms step_avg:155.07ms
step:1079/1480 train_time:165781ms step_avg:155.08ms
step:1080/1480 train_time:165941ms step_avg:155.09ms
step:1081/1480 train_time:166101ms step_avg:155.09ms
step:1082/1480 train_time:166260ms step_avg:155.09ms
step:1083/1480 train_time:166420ms step_avg:155.10ms
step:1084/1480 train_time:166579ms step_avg:155.10ms
step:1085/1480 train_time:166739ms step_avg:155.11ms
step:1086/1480 train_time:166899ms step_avg:155.11ms
step:1087/1480 train_time:167059ms step_avg:155.12ms
step:1088/1480 train_time:167220ms step_avg:155.12ms
step:1089/1480 train_time:167384ms step_avg:155.13ms
step:1090/1480 train_time:167548ms step_avg:155.14ms
step:1091/1480 train_time:167708ms step_avg:155.14ms
step:1092/1480 train_time:167869ms step_avg:155.15ms
step:1093/1480 train_time:168031ms step_avg:155.15ms
step:1094/1480 train_time:168192ms step_avg:155.16ms
step:1095/1480 train_time:168352ms step_avg:155.16ms
step:1096/1480 train_time:168516ms step_avg:155.17ms
step:1097/1480 train_time:168679ms step_avg:155.18ms
step:1098/1480 train_time:168840ms step_avg:155.18ms
step:1099/1480 train_time:169003ms step_avg:155.19ms
step:1100/1480 train_time:169165ms step_avg:155.20ms
step:1101/1480 train_time:169327ms step_avg:155.20ms
step:1102/1480 train_time:169491ms step_avg:155.21ms
step:1103/1480 train_time:169658ms step_avg:155.22ms
step:1104/1480 train_time:169819ms step_avg:155.23ms
step:1105/1480 train_time:169980ms step_avg:155.23ms
step:1106/1480 train_time:170140ms step_avg:155.24ms
step:1107/1480 train_time:170300ms step_avg:155.24ms
step:1108/1480 train_time:170460ms step_avg:155.25ms
step:1109/1480 train_time:170619ms step_avg:155.25ms
step:1110/1480 train_time:170779ms step_avg:155.25ms
step:1111/1480 train_time:170940ms step_avg:155.26ms
step:1112/1480 train_time:171100ms step_avg:155.26ms
step:1113/1480 train_time:171270ms step_avg:155.28ms
step:1114/1480 train_time:171434ms step_avg:155.28ms
step:1115/1480 train_time:171595ms step_avg:155.29ms
step:1116/1480 train_time:171755ms step_avg:155.29ms
step:1117/1480 train_time:171919ms step_avg:155.30ms
step:1118/1480 train_time:172084ms step_avg:155.31ms
step:1119/1480 train_time:172244ms step_avg:155.31ms
step:1120/1480 train_time:172404ms step_avg:155.32ms
step:1121/1480 train_time:172565ms step_avg:155.32ms
step:1122/1480 train_time:172726ms step_avg:155.33ms
step:1123/1480 train_time:172886ms step_avg:155.33ms
step:1124/1480 train_time:173050ms step_avg:155.34ms
step:1125/1480 train_time:173213ms step_avg:155.35ms
step:1125/1480 val_loss:3.3819 train_time:173288ms step_avg:155.42ms
step:1126/1480 train_time:173380ms step_avg:155.36ms
step:1127/1480 train_time:173539ms step_avg:155.36ms
step:1128/1480 train_time:173701ms step_avg:155.37ms
step:1129/1480 train_time:173862ms step_avg:155.37ms
step:1130/1480 train_time:174022ms step_avg:155.38ms
step:1131/1480 train_time:174190ms step_avg:155.39ms
step:1132/1480 train_time:174350ms step_avg:155.39ms
step:1133/1480 train_time:174515ms step_avg:155.40ms
step:1134/1480 train_time:174677ms step_avg:155.41ms
step:1135/1480 train_time:174839ms step_avg:155.41ms
step:1136/1480 train_time:175001ms step_avg:155.42ms
step:1137/1480 train_time:175162ms step_avg:155.42ms
step:1138/1480 train_time:175330ms step_avg:155.43ms
step:1139/1480 train_time:175497ms step_avg:155.44ms
step:1140/1480 train_time:175652ms step_avg:155.44ms
step:1141/1480 train_time:175818ms step_avg:155.45ms
step:1142/1480 train_time:175977ms step_avg:155.46ms
step:1143/1480 train_time:176142ms step_avg:155.46ms
step:1144/1480 train_time:176302ms step_avg:155.47ms
step:1145/1480 train_time:176461ms step_avg:155.47ms
step:1146/1480 train_time:176624ms step_avg:155.48ms
step:1147/1480 train_time:176787ms step_avg:155.49ms
step:1148/1480 train_time:176949ms step_avg:155.49ms
step:1149/1480 train_time:177113ms step_avg:155.50ms
step:1150/1480 train_time:177273ms step_avg:155.50ms
step:1151/1480 train_time:177436ms step_avg:155.51ms
step:1152/1480 train_time:177599ms step_avg:155.52ms
step:1153/1480 train_time:177764ms step_avg:155.52ms
step:1154/1480 train_time:177925ms step_avg:155.53ms
step:1155/1480 train_time:178086ms step_avg:155.53ms
step:1156/1480 train_time:178251ms step_avg:155.54ms
step:1157/1480 train_time:178415ms step_avg:155.55ms
step:1158/1480 train_time:178576ms step_avg:155.55ms
step:1159/1480 train_time:178736ms step_avg:155.56ms
step:1160/1480 train_time:178896ms step_avg:155.56ms
step:1161/1480 train_time:179057ms step_avg:155.57ms
step:1162/1480 train_time:179218ms step_avg:155.57ms
step:1163/1480 train_time:179380ms step_avg:155.58ms
step:1164/1480 train_time:179543ms step_avg:155.58ms
step:1165/1480 train_time:179703ms step_avg:155.59ms
step:1166/1480 train_time:179865ms step_avg:155.59ms
step:1167/1480 train_time:180026ms step_avg:155.60ms
step:1168/1480 train_time:180190ms step_avg:155.60ms
step:1169/1480 train_time:180351ms step_avg:155.61ms
step:1170/1480 train_time:180513ms step_avg:155.61ms
step:1171/1480 train_time:180676ms step_avg:155.62ms
step:1172/1480 train_time:180836ms step_avg:155.62ms
step:1173/1480 train_time:180997ms step_avg:155.63ms
step:1174/1480 train_time:181168ms step_avg:155.64ms
step:1175/1480 train_time:181330ms step_avg:155.65ms
step:1176/1480 train_time:181494ms step_avg:155.66ms
step:1177/1480 train_time:181661ms step_avg:155.66ms
step:1178/1480 train_time:181822ms step_avg:155.67ms
step:1179/1480 train_time:181981ms step_avg:155.67ms
step:1180/1480 train_time:182150ms step_avg:155.68ms
step:1181/1480 train_time:182313ms step_avg:155.69ms
step:1182/1480 train_time:182473ms step_avg:155.69ms
step:1183/1480 train_time:182634ms step_avg:155.70ms
step:1184/1480 train_time:182795ms step_avg:155.70ms
step:1185/1480 train_time:182959ms step_avg:155.71ms
step:1186/1480 train_time:183120ms step_avg:155.71ms
step:1187/1480 train_time:183292ms step_avg:155.73ms
step:1188/1480 train_time:183451ms step_avg:155.73ms
step:1189/1480 train_time:183614ms step_avg:155.74ms
step:1190/1480 train_time:183776ms step_avg:155.74ms
step:1191/1480 train_time:183940ms step_avg:155.75ms
step:1192/1480 train_time:184099ms step_avg:155.75ms
step:1193/1480 train_time:184258ms step_avg:155.76ms
step:1194/1480 train_time:184420ms step_avg:155.76ms
step:1195/1480 train_time:184583ms step_avg:155.77ms
step:1196/1480 train_time:184750ms step_avg:155.78ms
step:1197/1480 train_time:184914ms step_avg:155.78ms
step:1198/1480 train_time:185083ms step_avg:155.79ms
step:1199/1480 train_time:185245ms step_avg:155.80ms
step:1200/1480 train_time:185409ms step_avg:155.81ms
step:1201/1480 train_time:185569ms step_avg:155.81ms
step:1202/1480 train_time:185737ms step_avg:155.82ms
step:1203/1480 train_time:185903ms step_avg:155.83ms
step:1204/1480 train_time:186068ms step_avg:155.84ms
step:1205/1480 train_time:186230ms step_avg:155.84ms
step:1206/1480 train_time:186392ms step_avg:155.85ms
step:1207/1480 train_time:186552ms step_avg:155.85ms
step:1208/1480 train_time:186712ms step_avg:155.85ms
step:1209/1480 train_time:186875ms step_avg:155.86ms
step:1210/1480 train_time:187041ms step_avg:155.87ms
step:1211/1480 train_time:187205ms step_avg:155.87ms
step:1212/1480 train_time:187368ms step_avg:155.88ms
step:1213/1480 train_time:187533ms step_avg:155.89ms
step:1214/1480 train_time:187699ms step_avg:155.90ms
step:1215/1480 train_time:187864ms step_avg:155.90ms
step:1216/1480 train_time:188025ms step_avg:155.91ms
step:1217/1480 train_time:188190ms step_avg:155.92ms
step:1218/1480 train_time:188352ms step_avg:155.92ms
step:1219/1480 train_time:188519ms step_avg:155.93ms
step:1220/1480 train_time:188681ms step_avg:155.93ms
step:1221/1480 train_time:188841ms step_avg:155.94ms
step:1222/1480 train_time:189001ms step_avg:155.94ms
step:1223/1480 train_time:189164ms step_avg:155.95ms
step:1224/1480 train_time:189330ms step_avg:155.96ms
step:1225/1480 train_time:189494ms step_avg:155.96ms
step:1226/1480 train_time:189657ms step_avg:155.97ms
step:1227/1480 train_time:189820ms step_avg:155.97ms
step:1228/1480 train_time:189982ms step_avg:155.98ms
step:1229/1480 train_time:190145ms step_avg:155.98ms
step:1230/1480 train_time:190315ms step_avg:156.00ms
step:1231/1480 train_time:190480ms step_avg:156.00ms
step:1232/1480 train_time:190646ms step_avg:156.01ms
step:1233/1480 train_time:190807ms step_avg:156.02ms
step:1234/1480 train_time:190969ms step_avg:156.02ms
step:1235/1480 train_time:191133ms step_avg:156.03ms
step:1236/1480 train_time:191295ms step_avg:156.03ms
step:1237/1480 train_time:191455ms step_avg:156.04ms
step:1238/1480 train_time:191631ms step_avg:156.05ms
step:1239/1480 train_time:191793ms step_avg:156.06ms
step:1240/1480 train_time:191957ms step_avg:156.06ms
step:1241/1480 train_time:192123ms step_avg:156.07ms
step:1242/1480 train_time:192285ms step_avg:156.08ms
step:1243/1480 train_time:192448ms step_avg:156.08ms
step:1244/1480 train_time:192611ms step_avg:156.09ms
step:1245/1480 train_time:192774ms step_avg:156.09ms
step:1246/1480 train_time:192936ms step_avg:156.10ms
step:1247/1480 train_time:193098ms step_avg:156.10ms
step:1248/1480 train_time:193260ms step_avg:156.11ms
step:1249/1480 train_time:193420ms step_avg:156.11ms
step:1250/1480 train_time:193582ms step_avg:156.11ms
step:1250/1480 val_loss:3.3325 train_time:193658ms step_avg:156.18ms
step:1251/1480 train_time:193751ms step_avg:156.12ms
step:1252/1480 train_time:193914ms step_avg:156.13ms
step:1253/1480 train_time:194074ms step_avg:156.13ms
step:1254/1480 train_time:194236ms step_avg:156.14ms
step:1255/1480 train_time:194407ms step_avg:156.15ms
step:1256/1480 train_time:194571ms step_avg:156.16ms
step:1257/1480 train_time:194735ms step_avg:156.16ms
step:1258/1480 train_time:194899ms step_avg:156.17ms
step:1259/1480 train_time:195064ms step_avg:156.18ms
step:1260/1480 train_time:195224ms step_avg:156.18ms
step:1261/1480 train_time:195387ms step_avg:156.18ms
step:1262/1480 train_time:195551ms step_avg:156.19ms
step:1263/1480 train_time:195717ms step_avg:156.20ms
step:1264/1480 train_time:195877ms step_avg:156.20ms
step:1265/1480 train_time:196038ms step_avg:156.21ms
step:1266/1480 train_time:196203ms step_avg:156.21ms
step:1267/1480 train_time:196365ms step_avg:156.22ms
step:1268/1480 train_time:196527ms step_avg:156.22ms
step:1269/1480 train_time:196691ms step_avg:156.23ms
step:1270/1480 train_time:196853ms step_avg:156.23ms
step:1271/1480 train_time:197016ms step_avg:156.24ms
step:1272/1480 train_time:197176ms step_avg:156.24ms
step:1273/1480 train_time:197338ms step_avg:156.25ms
step:1274/1480 train_time:197502ms step_avg:156.25ms
step:1275/1480 train_time:197664ms step_avg:156.26ms
step:1276/1480 train_time:197824ms step_avg:156.26ms
step:1277/1480 train_time:197987ms step_avg:156.26ms
step:1278/1480 train_time:198147ms step_avg:156.27ms
step:1279/1480 train_time:198309ms step_avg:156.27ms
step:1280/1480 train_time:198475ms step_avg:156.28ms
step:1281/1480 train_time:198638ms step_avg:156.29ms
step:1282/1480 train_time:198799ms step_avg:156.29ms
step:1283/1480 train_time:198962ms step_avg:156.29ms
step:1284/1480 train_time:199127ms step_avg:156.30ms
step:1285/1480 train_time:199288ms step_avg:156.30ms
step:1286/1480 train_time:199449ms step_avg:156.31ms
step:1287/1480 train_time:199613ms step_avg:156.31ms
step:1288/1480 train_time:199774ms step_avg:156.32ms
step:1289/1480 train_time:199944ms step_avg:156.33ms
step:1290/1480 train_time:200112ms step_avg:156.34ms
step:1291/1480 train_time:200276ms step_avg:156.34ms
step:1292/1480 train_time:200442ms step_avg:156.35ms
step:1293/1480 train_time:200608ms step_avg:156.36ms
step:1294/1480 train_time:200770ms step_avg:156.36ms
step:1295/1480 train_time:200932ms step_avg:156.37ms
step:1296/1480 train_time:201095ms step_avg:156.37ms
step:1297/1480 train_time:201258ms step_avg:156.38ms
step:1298/1480 train_time:201422ms step_avg:156.38ms
step:1299/1480 train_time:201585ms step_avg:156.39ms
step:1300/1480 train_time:201747ms step_avg:156.39ms
step:1301/1480 train_time:201907ms step_avg:156.40ms
step:1302/1480 train_time:202073ms step_avg:156.40ms
step:1303/1480 train_time:202242ms step_avg:156.41ms
step:1304/1480 train_time:202407ms step_avg:156.42ms
step:1305/1480 train_time:202568ms step_avg:156.42ms
step:1306/1480 train_time:202733ms step_avg:156.43ms
step:1307/1480 train_time:202895ms step_avg:156.43ms
step:1308/1480 train_time:203057ms step_avg:156.44ms
step:1309/1480 train_time:203223ms step_avg:156.45ms
step:1310/1480 train_time:203384ms step_avg:156.45ms
step:1311/1480 train_time:203545ms step_avg:156.45ms
step:1312/1480 train_time:203709ms step_avg:156.46ms
step:1313/1480 train_time:203870ms step_avg:156.46ms
step:1314/1480 train_time:204036ms step_avg:156.47ms
step:1315/1480 train_time:204201ms step_avg:156.48ms
step:1316/1480 train_time:204361ms step_avg:156.48ms
step:1317/1480 train_time:204522ms step_avg:156.48ms
step:1318/1480 train_time:204690ms step_avg:156.49ms
step:1319/1480 train_time:204855ms step_avg:156.50ms
step:1320/1480 train_time:205023ms step_avg:156.51ms
step:1321/1480 train_time:205186ms step_avg:156.51ms
step:1322/1480 train_time:205355ms step_avg:156.52ms
step:1323/1480 train_time:205519ms step_avg:156.53ms
step:1324/1480 train_time:205683ms step_avg:156.53ms
step:1325/1480 train_time:205852ms step_avg:156.54ms
step:1326/1480 train_time:206018ms step_avg:156.55ms
step:1327/1480 train_time:206181ms step_avg:156.55ms
step:1328/1480 train_time:206344ms step_avg:156.56ms
step:1329/1480 train_time:206527ms step_avg:156.58ms
step:1330/1480 train_time:206693ms step_avg:156.59ms
step:1331/1480 train_time:206855ms step_avg:156.59ms
step:1332/1480 train_time:207018ms step_avg:156.59ms
step:1333/1480 train_time:207184ms step_avg:156.60ms
step:1334/1480 train_time:207348ms step_avg:156.61ms
step:1335/1480 train_time:207507ms step_avg:156.61ms
step:1336/1480 train_time:207677ms step_avg:156.62ms
step:1337/1480 train_time:207846ms step_avg:156.63ms
step:1338/1480 train_time:208009ms step_avg:156.63ms
step:1339/1480 train_time:208172ms step_avg:156.64ms
step:1340/1480 train_time:208337ms step_avg:156.64ms
step:1341/1480 train_time:208499ms step_avg:156.65ms
step:1342/1480 train_time:208665ms step_avg:156.66ms
step:1343/1480 train_time:208826ms step_avg:156.66ms
step:1344/1480 train_time:208988ms step_avg:156.66ms
step:1345/1480 train_time:209158ms step_avg:156.67ms
step:1346/1480 train_time:209320ms step_avg:156.68ms
step:1347/1480 train_time:209483ms step_avg:156.68ms
step:1348/1480 train_time:209647ms step_avg:156.69ms
step:1349/1480 train_time:209810ms step_avg:156.69ms
step:1350/1480 train_time:209977ms step_avg:156.70ms
step:1351/1480 train_time:210140ms step_avg:156.70ms
step:1352/1480 train_time:210303ms step_avg:156.71ms
step:1353/1480 train_time:210468ms step_avg:156.72ms
step:1354/1480 train_time:210633ms step_avg:156.72ms
step:1355/1480 train_time:210794ms step_avg:156.72ms
step:1356/1480 train_time:210958ms step_avg:156.73ms
step:1357/1480 train_time:211123ms step_avg:156.74ms
step:1358/1480 train_time:211289ms step_avg:156.74ms
step:1359/1480 train_time:211453ms step_avg:156.75ms
step:1360/1480 train_time:211619ms step_avg:156.75ms
step:1361/1480 train_time:211787ms step_avg:156.76ms
step:1362/1480 train_time:211950ms step_avg:156.77ms
step:1363/1480 train_time:212118ms step_avg:156.78ms
step:1364/1480 train_time:212281ms step_avg:156.78ms
step:1365/1480 train_time:212441ms step_avg:156.78ms
step:1366/1480 train_time:212605ms step_avg:156.79ms
step:1367/1480 train_time:212768ms step_avg:156.79ms
step:1368/1480 train_time:212934ms step_avg:156.80ms
step:1369/1480 train_time:213103ms step_avg:156.81ms
step:1370/1480 train_time:213268ms step_avg:156.81ms
step:1371/1480 train_time:213431ms step_avg:156.82ms
step:1372/1480 train_time:213599ms step_avg:156.83ms
step:1373/1480 train_time:213761ms step_avg:156.83ms
step:1374/1480 train_time:213927ms step_avg:156.84ms
step:1375/1480 train_time:214089ms step_avg:156.84ms
step:1375/1480 val_loss:3.2941 train_time:214162ms step_avg:156.90ms
step:1376/1480 train_time:214254ms step_avg:156.85ms
step:1377/1480 train_time:214418ms step_avg:156.85ms
step:1378/1480 train_time:214580ms step_avg:156.86ms
step:1379/1480 train_time:214745ms step_avg:156.86ms
step:1380/1480 train_time:214909ms step_avg:156.87ms
step:1381/1480 train_time:215077ms step_avg:156.88ms
step:1382/1480 train_time:215240ms step_avg:156.88ms
step:1383/1480 train_time:215403ms step_avg:156.88ms
step:1384/1480 train_time:215570ms step_avg:156.89ms
step:1385/1480 train_time:215730ms step_avg:156.89ms
step:1386/1480 train_time:215893ms step_avg:156.90ms
step:1387/1480 train_time:216060ms step_avg:156.91ms
step:1388/1480 train_time:216221ms step_avg:156.91ms
step:1389/1480 train_time:216386ms step_avg:156.91ms
step:1390/1480 train_time:216548ms step_avg:156.92ms
step:1391/1480 train_time:216711ms step_avg:156.92ms
step:1392/1480 train_time:216875ms step_avg:156.93ms
step:1393/1480 train_time:217037ms step_avg:156.93ms
step:1394/1480 train_time:217199ms step_avg:156.94ms
step:1395/1480 train_time:217361ms step_avg:156.94ms
step:1396/1480 train_time:217523ms step_avg:156.94ms
step:1397/1480 train_time:217683ms step_avg:156.95ms
step:1398/1480 train_time:217844ms step_avg:156.95ms
step:1399/1480 train_time:218005ms step_avg:156.95ms
step:1400/1480 train_time:218174ms step_avg:156.96ms
step:1401/1480 train_time:218334ms step_avg:156.96ms
step:1402/1480 train_time:218496ms step_avg:156.97ms
step:1403/1480 train_time:218661ms step_avg:156.97ms
step:1404/1480 train_time:218824ms step_avg:156.98ms
step:1405/1480 train_time:218987ms step_avg:156.98ms
step:1406/1480 train_time:219152ms step_avg:156.99ms
step:1407/1480 train_time:219313ms step_avg:156.99ms
step:1408/1480 train_time:219476ms step_avg:156.99ms
step:1409/1480 train_time:219649ms step_avg:157.00ms
step:1410/1480 train_time:219812ms step_avg:157.01ms
step:1411/1480 train_time:219975ms step_avg:157.01ms
step:1412/1480 train_time:220138ms step_avg:157.02ms
step:1413/1480 train_time:220301ms step_avg:157.02ms
step:1414/1480 train_time:220464ms step_avg:157.03ms
step:1415/1480 train_time:220629ms step_avg:157.03ms
step:1416/1480 train_time:220802ms step_avg:157.04ms
step:1417/1480 train_time:220967ms step_avg:157.05ms
step:1418/1480 train_time:221130ms step_avg:157.05ms
step:1419/1480 train_time:221297ms step_avg:157.06ms
step:1420/1480 train_time:221461ms step_avg:157.06ms
step:1421/1480 train_time:221625ms step_avg:157.07ms
step:1422/1480 train_time:221791ms step_avg:157.08ms
step:1423/1480 train_time:221954ms step_avg:157.08ms
step:1424/1480 train_time:222120ms step_avg:157.09ms
step:1425/1480 train_time:222291ms step_avg:157.10ms
step:1426/1480 train_time:222456ms step_avg:157.10ms
step:1427/1480 train_time:222622ms step_avg:157.11ms
step:1428/1480 train_time:222783ms step_avg:157.11ms
step:1429/1480 train_time:222942ms step_avg:157.11ms
step:1430/1480 train_time:223107ms step_avg:157.12ms
step:1431/1480 train_time:223274ms step_avg:157.12ms
step:1432/1480 train_time:223441ms step_avg:157.13ms
step:1433/1480 train_time:223611ms step_avg:157.14ms
step:1434/1480 train_time:223780ms step_avg:157.15ms
step:1435/1480 train_time:223945ms step_avg:157.15ms
step:1436/1480 train_time:224111ms step_avg:157.16ms
step:1437/1480 train_time:224274ms step_avg:157.16ms
step:1438/1480 train_time:224436ms step_avg:157.17ms
step:1439/1480 train_time:224604ms step_avg:157.18ms
step:1440/1480 train_time:224767ms step_avg:157.18ms
step:1441/1480 train_time:224932ms step_avg:157.18ms
step:1442/1480 train_time:225098ms step_avg:157.19ms
step:1443/1480 train_time:225271ms step_avg:157.20ms
step:1444/1480 train_time:225435ms step_avg:157.21ms
step:1445/1480 train_time:225599ms step_avg:157.21ms
step:1446/1480 train_time:225764ms step_avg:157.22ms
step:1447/1480 train_time:225933ms step_avg:157.23ms
step:1448/1480 train_time:226097ms step_avg:157.23ms
step:1449/1480 train_time:226260ms step_avg:157.23ms
step:1450/1480 train_time:226424ms step_avg:157.24ms
step:1451/1480 train_time:226588ms step_avg:157.24ms
step:1452/1480 train_time:226753ms step_avg:157.25ms
step:1453/1480 train_time:226916ms step_avg:157.25ms
step:1454/1480 train_time:227078ms step_avg:157.26ms
step:1455/1480 train_time:227246ms step_avg:157.26ms
step:1456/1480 train_time:227410ms step_avg:157.27ms
step:1457/1480 train_time:227573ms step_avg:157.27ms
step:1458/1480 train_time:227736ms step_avg:157.28ms
step:1459/1480 train_time:227901ms step_avg:157.28ms
step:1460/1480 train_time:228063ms step_avg:157.28ms
step:1461/1480 train_time:228228ms step_avg:157.29ms
step:1462/1480 train_time:228393ms step_avg:157.30ms
step:1463/1480 train_time:228558ms step_avg:157.30ms
step:1464/1480 train_time:228724ms step_avg:157.31ms
step:1465/1480 train_time:228888ms step_avg:157.31ms
step:1466/1480 train_time:229052ms step_avg:157.32ms
step:1467/1480 train_time:229218ms step_avg:157.32ms
step:1468/1480 train_time:229381ms step_avg:157.33ms
step:1469/1480 train_time:229545ms step_avg:157.33ms
step:1470/1480 train_time:229714ms step_avg:157.34ms
step:1471/1480 train_time:229884ms step_avg:157.35ms
step:1472/1480 train_time:230054ms step_avg:157.36ms
step:1473/1480 train_time:230217ms step_avg:157.36ms
step:1474/1480 train_time:230383ms step_avg:157.37ms
step:1475/1480 train_time:230552ms step_avg:157.37ms
step:1476/1480 train_time:230716ms step_avg:157.38ms
step:1477/1480 train_time:230884ms step_avg:157.39ms
step:1478/1480 train_time:231054ms step_avg:157.39ms
step:1479/1480 train_time:231221ms step_avg:157.40ms
step:1480/1480 train_time:231385ms step_avg:157.40ms
step:1480/1480 val_loss:3.2752 train_time:231460ms step_avg:157.46ms
peak memory consumption: 34239 MiB