9323a20a-8f7f-4549-8997-dde75dfc6ec6.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
import contextlib
from dataclasses import dataclass
from pathlib import Path

import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.nn.attention.flex_attention import BlockMask, flex_attention #KoszarskyB

# -----------------------------------------------------------------------------
# Muon optimizer

@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert len(G.shape) == 2
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    X /= (X.norm() + eps) # ensure top singular value <= 1
    if G.size(0) > G.size(1):
        X = X.T
    for _ in range(steps):
        A = X @ X.T
        B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    if G.size(0) > G.size(1):
        X = X.T
    return X

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven't tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        ns_steps: The number of Newton-Schulz iteration steps to use.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True, ns_steps=5):
        self.world_size = int(os.environ['WORLD_SIZE'])
        self.rank = int(os.environ['RANK'])
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, ns_steps=ns_steps)
        params = list(params)
        assert all(isinstance(p, torch.Tensor) for p in params)
        sizes = {p.numel() for p in params}
        param_groups = [
            {
                'params': [p for p in params if p.numel() == size],
                'update_buffer': [
                    torch.empty(size, device='cuda', dtype=torch.bfloat16)
                    for _ in range(self.world_size)
                ],
            }
            for size in sizes
        ]
        super().__init__(param_groups, defaults)

    def step(self):

        for group in self.param_groups:

            lr = group['lr']
            momentum = group['momentum']
            nesterov = group['nesterov']
            ns_steps = group['ns_steps']
            update_buffers = group['update_buffer']
            # generate weight updates in distributed fashion
            params = group['params']
            assert len(params) % self.world_size == 0
            handle = None
            params_world = None
            def update_prev():
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffers):
                    p_world.data.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.world_size]:
                p = params[base_i + self.rank]
                g = p.grad
                assert g is not None
                state = self.state[p]
                if 'momentum_buffer' not in state:
                    state['momentum_buffer'] = torch.zeros_like(g)
                buf = state['momentum_buffer']
                buf.lerp_(g, 1 - momentum)
                g = g.lerp_(buf, momentum) if nesterov else buf
                g = zeropower_via_newtonschulz5(g, steps=ns_steps).flatten()
                update_prev()
                handle = dist.all_gather(update_buffers, g, async_op=True)
                params_world = params[base_i : base_i + self.world_size]
            update_prev()

# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):

    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=False)

    def forward(self, x):
        return F.linear(x, self.weight.to(x.dtype))

class Rotary(torch.nn.Module):

    def __init__(self, dim, base=10000):
        super().__init__()
        self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None

    def forward(self, x):
        seq_len = x.shape[1]
        if seq_len != self.seq_len_cached:
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.outer(t, self.inv_freq)
            self.seq_len_cached = seq_len
            self.cos_cached = freqs.cos()
            self.sin_cached = freqs.sin()
        cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
        # apply_rotary_emb(x, cos, sin)
        x1, x2 = x.chunk(2, dim=3)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x)

class CausalSelfAttention(nn.Module):

    def __init__(self, dim, num_heads):
        super().__init__()
        assert dim % num_heads == 0
        self.num_heads = num_heads
        self.c_q = CastedLinear(dim, dim)
        self.c_k = CastedLinear(dim, dim)
        self.c_v = CastedLinear(dim, dim)
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5]))
        self.rotary = Rotary(dim // num_heads) # dim // num_heads = head_dim
        self.c_proj = CastedLinear(dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x, vi, block_mask):
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q = self.c_q(x).view(B, T, self.num_heads, -1)
        k = self.c_k(x).view(B, T, self.num_heads, -1)
        v = self.c_v(x).view(B, T, self.num_heads, -1)
        v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @KoszarskyB & @Grad62304977
        q, k = norm(q), norm(k) # QK norm @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask, enable_gqa=True)
        y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):

    def __init__(self, dim):
        super().__init__()
        self.c_fc   = CastedLinear(dim, 4 * dim)
        self.c_proj = CastedLinear(4 * dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x):
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):

    def __init__(self, config):
        super().__init__()
        self.attn = CausalSelfAttention(config.model_dim, config.num_heads)
        self.mlp = MLP(config.model_dim)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x, vi, x0, block_mask):
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        x = x + self.attn(norm(x), vi, block_mask)
        x = x + self.mlp(norm(x))
        return x

class ValueEmbedding(nn.Module):
    def __init__(self, config: "GPTConfig"):
        super().__init__()
        self.__setattr__
        self.embed = nn.ModuleList([
            nn.Embedding(config.vocab_size, config.model_dim)
            for _ in range(6)
        ])

    def forward(self, inputs) -> "list[torch.Tensor]":
        ve = [emb(inputs) for emb in self.embed]
        ve += reversed(ve)
        return ve


# -----------------------------------------------------------------------------
# The main GPT-2 model

@dataclass
class GPTConfig:
    vocab_size : int = 50304
    num_layers : int = 12
    num_heads : int = 6 # head dim 128 suggested by @Grad62304977
    model_dim : int = 768

class GPT(nn.Module):

    def __init__(self, config: GPTConfig):
        super().__init__()
        self.num_layers = config.num_layers

        # U-net design by @brendanh0gan
        self.num_encoder_layers = config.num_layers // 2 # Half of the layers for encoder
        self.num_decoder_layers = config.num_layers - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))

        self.embed = nn.Embedding(config.vocab_size, config.model_dim)
        self.blocks = nn.ModuleList([Block(config) for _ in range(config.num_layers)])
        # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
        # U-net structure on token value embeddings by @leloykun
        self.value_embeds = ValueEmbedding(config)
        self.lm_head = CastedLinear(config.model_dim, config.vocab_size)
        self.lm_head.weight.data.zero_() # @Grad62304977

    def forward(
        self,
        inputs: torch.Tensor,
        targets: torch.Tensor,
        sliding_window_num_blocks: torch.Tensor,
    ):
        BLOCK_SIZE = 128
        assert inputs.ndim == 1
        docs = (inputs == 50256).cumsum(0)
        docs_low = docs.view(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.view(-1, BLOCK_SIZE)[:, -1].contiguous()

        def document_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            return causal_mask & document_mask

        def dense_to_ordered(dense_mask: torch.Tensor):
            num_blocks = dense_mask.sum(dim=-1, dtype=torch.int32)
            indices = dense_mask.argsort(dim=-1, descending=True, stable=True).to(torch.int32)
            return num_blocks[None, None].contiguous(), indices[None, None].contiguous()

        def create_doc_swc_block_mask(sliding_window_num_blocks: torch.Tensor):
            kv_idx = block_idx = torch.arange(512, dtype=torch.int32, device="cuda")
            q_idx = block_idx[:, None]
            causal_bm = q_idx >= kv_idx
            causal_full_bm = q_idx > kv_idx
            window_bm = q_idx - kv_idx < sliding_window_num_blocks
            window_full_bm = window_bm
            # document_bm = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
            document_bm = (docs_low[:, None] <= docs_high) & (docs_low <= docs_high[:, None])
            document_full_bm = (docs_low[:, None] == docs_high) & (docs_low == docs_high[:, None])
            nonzero_bm = causal_bm & window_bm & document_bm
            full_bm  = causal_full_bm & window_full_bm & document_full_bm
            kv_num_blocks, kv_indices = dense_to_ordered(nonzero_bm ^ full_bm)
            full_kv_num_blocks, full_kv_indices = dense_to_ordered(full_bm)
            return BlockMask.from_kv_blocks(
                kv_num_blocks,
                kv_indices,
                full_kv_num_blocks,
                full_kv_indices,
                BLOCK_SIZE=BLOCK_SIZE,
                mask_mod=document_causal,
            )

        block_mask = create_doc_swc_block_mask(sliding_window_num_blocks)

        # forward the GPT model itself
        x = self.embed(inputs[None]) # token embeddings of shape (b, t, model_dim)
        x = norm(x) # @Grad62304977
        x0 = x
        ve = self.value_embeds(inputs)
        ve_enc, ve_dec = ve[:self.num_encoder_layers], ve[self.num_encoder_layers:]

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        for i in range(self.num_encoder_layers):
            x = self.blocks[i](x, ve_enc[i], x0, block_mask)
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            # U-net structure on token value embeddings by @leloykun
            x = self.blocks[self.num_encoder_layers + i](x, ve_dec[i], x0, block_mask)

        x = norm(x)
        logits = self.lm_head(x)
        logits = 30 * torch.tanh(logits / 30) # @Grad62304977
        logits = logits.float()
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1))
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _peek_data_shard(file: Path):
    # only reads the header, returns header data
    # header is 256 int32
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    return int(header[2]) # number of tokens (claimed)

def _load_data_shard(path: Path, num_tokens):
    with path.open("rb", buffering=0) as f:
        tokens = torch.empty(num_tokens, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy())
        assert nbytes == 2 * num_tokens, "number of tokens read does not match header?"
    return tokens

class DistributedDataLoader:
    def __init__(self, filename_pattern, seq_len, process_rank, num_processes):
        self.process_rank = process_rank
        self.num_processes = num_processes
        self.seq_len = seq_len

        # glob files that match the pattern
        self.files = sorted(Path.cwd().glob(filename_pattern))
        assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"

        # load and validate all data shards, count number of tokens in total
        self.files_num_tokens = [_peek_data_shard(file) for file in self.files]
        assert min(self.files_num_tokens) >= num_processes * seq_len + 1
        self.total_num_tokens = sum(self.files_num_tokens)

        self.reset()

    def reset(self):
        self.current_shard = -1
        self.advance()

    def advance(self): # advance to next data shard
        self.current_shard = (self.current_shard + 1) % len(self.files)
        self.current_position = self.process_rank * self.seq_len
        self.tokens = _load_data_shard(self.files[self.current_shard], self.files_num_tokens[self.current_shard])

    def next_batch(self):
        batch_size = self.seq_len * self.num_processes
        buf = self.tokens[self.current_position:self.current_position+self.seq_len+1]
        # host side async is sufficient;
        # no performance improvement was observed when introducing a separate stream.
        inputs = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
        targets = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
        # advance current position and load next shard if necessary
        self.current_position += batch_size
        if self.current_position + batch_size + 1 >= len(self.tokens):
            self.advance()
        return inputs, targets

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data hyperparams
    input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
    input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
    # optimization hyperparams
    batch_size : int = 8 # batch size, in sequences, across all devices
    sequence_length : int = 64*1024 # sequence length, in tokens
    num_iterations : int = 1480 # number of iterations to run
    warmup_iters : int = 0
    cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
    weight_decay : float = 0
    # evaluation and logging hyperparams
    val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
    val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()

# set up DDP (distributed data parallel). torchrun sets this env variable
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
assert torch.cuda.is_available()
device = torch.device(f"cuda:{ddp_local_rank}")
torch.cuda.set_device(device)
print(f"using device: {device}")
dist.init_process_group(backend='nccl', device_id=device)
dist.barrier()
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = uuid.uuid4()
    logdir = Path("logs") / f"{run_id}"
    logdir.mkdir(exist_ok=True)
    logfile = Path("logs") / f"{run_id}.txt"
    print(logfile.stem)
    # create the log file
    with logfile.open("w") as f:
        # begin the log by printing this file (the Python code)
        print(code, file=f)
        print("=" * 100, file=f)
def print0(s, logonly=False):
    if master_process:
        with logfile.open("a") as f:
            if not logonly:
                print(s)
            print(s, file=f)
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running python {sys.version}")
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)

# calculate the number of steps to take in the val loop.
assert args.val_tokens % (args.sequence_length * ddp_world_size) == 0
val_steps = args.val_tokens // (args.sequence_length * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size

# load tokens
train_loader = DistributedDataLoader(args.input_bin, args.sequence_length, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, args.sequence_length, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.total_num_tokens} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.total_num_tokens} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
inputs_train, targets_train = train_loader.next_batch()

# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, num_layers=12, num_heads=6, model_dim=768))
model = model.cuda().bfloat16()
for m in model.modules():
    if isinstance(m, CastedLinear):
        m.float()
config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank], broadcast_buffers=False, gradient_as_bucket_view=True)
raw_model = model.module # always contains the "raw" unwrapped model

# init the optimizer(s)
embed_params = [*raw_model.embed.parameters(), *raw_model.value_embeds.parameters()]
optimizer1 = torch.optim.Adam(embed_params, lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.blocks.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
    assert it <= args.num_iterations
    # 1) linear warmup for warmup_iters steps
    if it < args.warmup_iters:
        return (it+1) / args.warmup_iters
    # 2) constant lr for a while
    elif it < args.num_iterations - args.cooldown_iters:
        return 1.0
    # 3) linear cooldown
    else:
        decay_ratio = (args.num_iterations - it) / args.cooldown_iters
        return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]

sliding_window_num_blocks = torch.tensor(1, dtype=torch.int32, device="cuda")
sw_num_blocks_prev = 1
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
    last_step = (step == args.num_iterations)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Linearly increase the sliding window size over training in chunks of 64 from 64 -> 1792. By @fernbear.bsky.social
    frac_done = step / args.num_iterations # training progress
    sw_num_blocks = int(((1 - frac_done) * 64 + frac_done * 1792 + 64) // 128)
    if sw_num_blocks != sw_num_blocks_prev:
        sliding_window_num_blocks.copy_(sw_num_blocks, non_blocking=True)
        sw_num_blocks_prev = sw_num_blocks

    # once in a while evaluate the validation dataset
    if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # run validation batches
        model.eval()
        val_loader.reset()
        val_loss = 0.0
        for _ in range(val_steps):
            with torch.no_grad():
                inputs_val, targets_val = val_loader.next_batch()
                val_loss += model(inputs_val, targets_val, sliding_window_num_blocks)
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        val_loss /= val_steps
        # log val loss to console and to logfile
        print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # save the state of the training process
        log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
        torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    # bit confusing: we want to make sure to eval on 0th iteration
    # but also after the very last iteration. so we loop for step <= num_iterations
    # instead of just < num_iterations (one extra due to <=), only to do
    # the validation/sampling one last time, and then we break right here as we're done.
    if last_step:
        break

    # --------------- TRAINING SECTION BEGIN -----------------
    model.train()
    for i in range(1, train_accumulation_steps + 1):
        with contextlib.ExitStack() as stack:
            if i < train_accumulation_steps: # there's no need to sync gradients every accumulation step
                stack.enter_context(model.no_sync())
            if step >= 5:
                stack.enter_context(torch.compiler.set_stance(skip_guard_eval_unsafe=True))
            model(inputs_train, targets_train, sliding_window_num_blocks).backward()
            inputs_train, targets_train = train_loader.next_batch()
    if train_accumulation_steps != 1:
        for p in model.parameters():
            p.grad /= train_accumulation_steps
    # momentum warmup for Muon
    frac = min(step/300, 1)
    for group in optimizer3.param_groups:
        group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # --------------- TRAINING SECTION END -------------------
    # everything that follows now is just diagnostics, prints, logging, etc.
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")

print0(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")

# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()

====================================================================================================
Running python 3.10.12 (main, Sep 11 2024, 15:47:36) [GCC 11.4.0]
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Wed Dec 11 07:43:03 2024       
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.183.06             Driver Version: 535.183.06   CUDA Version: 12.2     |
|-----------------------------------------+----------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
|                                         |                      |               MIG M. |
|=========================================+======================+======================|
|   0  NVIDIA H100 80GB HBM3          On  | 00000000:19:00.0 Off |                    0 |
| N/A   38C    P0             126W / 700W |   7084MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  | 00000000:3B:00.0 Off |                    0 |
| N/A   30C    P0             116W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  | 00000000:4C:00.0 Off |                    0 |
| N/A   28C    P0             112W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  | 00000000:5D:00.0 Off |                    0 |
| N/A   36C    P0             115W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  | 00000000:9B:00.0 Off |                    0 |
| N/A   38C    P0             120W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  | 00000000:BB:00.0 Off |                    0 |
| N/A   30C    P0             118W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  | 00000000:CB:00.0 Off |                    0 |
| N/A   36C    P0             120W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  | 00000000:DB:00.0 Off |                    0 |
| N/A   30C    P0             118W / 700W |   3211MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
                                                                                         
+---------------------------------------------------------------------------------------+
| Processes:                                                                            |
|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
|        ID   ID                                                             Usage      |
|=======================================================================================|
+---------------------------------------------------------------------------------------+

====================================================================================================
Training DataLoader: total number of tokens: 1000000000 across 10 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:28954ms step_avg:nanms
step:2/1480 train_time:29058ms step_avg:nanms
step:3/1480 train_time:29181ms step_avg:nanms
step:4/1480 train_time:29321ms step_avg:nanms
step:5/1480 train_time:29462ms step_avg:nanms
step:6/1480 train_time:29603ms step_avg:nanms
step:7/1480 train_time:29745ms step_avg:nanms
step:8/1480 train_time:29888ms step_avg:nanms
step:9/1480 train_time:30030ms step_avg:nanms
step:10/1480 train_time:30179ms step_avg:nanms
step:11/1480 train_time:141ms step_avg:nanms
step:12/1480 train_time:279ms step_avg:nanms
step:13/1480 train_time:421ms step_avg:140.41ms
step:14/1480 train_time:563ms step_avg:140.79ms
step:15/1480 train_time:706ms step_avg:141.13ms
step:16/1480 train_time:848ms step_avg:141.39ms
step:17/1480 train_time:993ms step_avg:141.92ms
step:18/1480 train_time:1135ms step_avg:141.86ms
step:19/1480 train_time:1278ms step_avg:142.04ms
step:20/1480 train_time:1421ms step_avg:142.07ms
step:21/1480 train_time:1564ms step_avg:142.19ms
step:22/1480 train_time:1708ms step_avg:142.31ms
step:23/1480 train_time:1850ms step_avg:142.34ms
step:24/1480 train_time:1994ms step_avg:142.41ms
step:25/1480 train_time:2136ms step_avg:142.38ms
step:26/1480 train_time:2277ms step_avg:142.32ms
step:27/1480 train_time:2420ms step_avg:142.37ms
step:28/1480 train_time:2563ms step_avg:142.37ms
step:29/1480 train_time:2706ms step_avg:142.44ms
step:30/1480 train_time:2849ms step_avg:142.45ms
step:31/1480 train_time:2992ms step_avg:142.46ms
step:32/1480 train_time:3133ms step_avg:142.40ms
step:33/1480 train_time:3275ms step_avg:142.40ms
step:34/1480 train_time:3418ms step_avg:142.43ms
step:35/1480 train_time:3562ms step_avg:142.47ms
step:36/1480 train_time:3705ms step_avg:142.48ms
step:37/1480 train_time:4206ms step_avg:155.76ms
step:38/1480 train_time:4305ms step_avg:153.73ms
step:39/1480 train_time:4828ms step_avg:166.48ms
step:40/1480 train_time:5332ms step_avg:177.74ms
step:41/1480 train_time:5434ms step_avg:175.28ms
step:42/1480 train_time:5575ms step_avg:174.22ms
step:43/1480 train_time:5717ms step_avg:173.24ms
step:44/1480 train_time:5858ms step_avg:172.30ms
step:45/1480 train_time:6001ms step_avg:171.47ms
step:46/1480 train_time:6143ms step_avg:170.63ms
step:47/1480 train_time:6288ms step_avg:169.95ms
step:48/1480 train_time:6435ms step_avg:169.34ms
step:49/1480 train_time:6579ms step_avg:168.70ms
step:50/1480 train_time:6722ms step_avg:168.06ms
step:51/1480 train_time:6865ms step_avg:167.45ms
step:52/1480 train_time:7009ms step_avg:166.88ms
step:53/1480 train_time:7152ms step_avg:166.32ms
step:54/1480 train_time:7294ms step_avg:165.78ms
step:55/1480 train_time:7436ms step_avg:165.23ms
step:56/1480 train_time:7577ms step_avg:164.73ms
step:57/1480 train_time:7720ms step_avg:164.25ms
step:58/1480 train_time:7862ms step_avg:163.80ms
step:59/1480 train_time:8005ms step_avg:163.37ms
step:60/1480 train_time:8148ms step_avg:162.96ms
step:61/1480 train_time:8291ms step_avg:162.58ms
step:62/1480 train_time:8433ms step_avg:162.17ms
step:63/1480 train_time:8576ms step_avg:161.81ms
step:64/1480 train_time:8718ms step_avg:161.45ms
step:65/1480 train_time:8860ms step_avg:161.10ms
step:66/1480 train_time:9005ms step_avg:160.80ms
step:67/1480 train_time:9148ms step_avg:160.49ms
step:68/1480 train_time:9292ms step_avg:160.21ms
step:69/1480 train_time:9433ms step_avg:159.88ms
step:70/1480 train_time:9575ms step_avg:159.58ms
step:71/1480 train_time:9717ms step_avg:159.29ms
step:72/1480 train_time:9860ms step_avg:159.03ms
step:73/1480 train_time:10001ms step_avg:158.75ms
step:74/1480 train_time:10144ms step_avg:158.49ms
step:75/1480 train_time:10287ms step_avg:158.27ms
step:76/1480 train_time:10430ms step_avg:158.04ms
step:77/1480 train_time:10575ms step_avg:157.83ms
step:78/1480 train_time:10716ms step_avg:157.59ms
step:79/1480 train_time:10859ms step_avg:157.38ms
step:80/1480 train_time:11000ms step_avg:157.14ms
step:81/1480 train_time:11143ms step_avg:156.94ms
step:82/1480 train_time:11285ms step_avg:156.74ms
step:83/1480 train_time:11428ms step_avg:156.55ms
step:84/1480 train_time:11570ms step_avg:156.36ms
step:85/1480 train_time:11713ms step_avg:156.17ms
step:86/1480 train_time:11855ms step_avg:155.99ms
step:87/1480 train_time:11998ms step_avg:155.82ms
step:88/1480 train_time:12140ms step_avg:155.64ms
step:89/1480 train_time:12284ms step_avg:155.50ms
step:90/1480 train_time:12429ms step_avg:155.36ms
step:91/1480 train_time:12572ms step_avg:155.20ms
step:92/1480 train_time:12715ms step_avg:155.06ms
step:93/1480 train_time:12857ms step_avg:154.91ms
step:94/1480 train_time:13000ms step_avg:154.76ms
step:95/1480 train_time:13142ms step_avg:154.61ms
step:96/1480 train_time:13284ms step_avg:154.46ms
step:97/1480 train_time:13426ms step_avg:154.32ms
step:98/1480 train_time:13570ms step_avg:154.20ms
step:99/1480 train_time:13712ms step_avg:154.07ms
step:100/1480 train_time:13855ms step_avg:153.95ms
step:101/1480 train_time:14000ms step_avg:153.85ms
step:102/1480 train_time:14140ms step_avg:153.70ms
step:103/1480 train_time:14283ms step_avg:153.58ms
step:104/1480 train_time:14427ms step_avg:153.48ms
step:105/1480 train_time:14572ms step_avg:153.39ms
step:106/1480 train_time:14713ms step_avg:153.26ms
step:107/1480 train_time:14855ms step_avg:153.15ms
step:108/1480 train_time:14998ms step_avg:153.04ms
step:109/1480 train_time:15142ms step_avg:152.95ms
step:110/1480 train_time:15285ms step_avg:152.85ms
step:111/1480 train_time:15428ms step_avg:152.76ms
step:112/1480 train_time:15574ms step_avg:152.69ms
step:113/1480 train_time:15719ms step_avg:152.61ms
step:114/1480 train_time:15865ms step_avg:152.55ms
step:115/1480 train_time:16011ms step_avg:152.49ms
step:116/1480 train_time:16156ms step_avg:152.41ms
step:117/1480 train_time:16300ms step_avg:152.34ms
step:118/1480 train_time:16446ms step_avg:152.28ms
step:119/1480 train_time:16593ms step_avg:152.23ms
step:120/1480 train_time:16739ms step_avg:152.17ms
step:121/1480 train_time:16885ms step_avg:152.12ms
step:122/1480 train_time:17031ms step_avg:152.06ms
step:123/1480 train_time:17177ms step_avg:152.01ms
step:124/1480 train_time:17322ms step_avg:151.95ms
step:125/1480 train_time:17469ms step_avg:151.90ms
step:125/1480 val_loss:4.4157 train_time:17533ms step_avg:152.46ms
step:126/1480 train_time:17628ms step_avg:151.97ms
step:127/1480 train_time:17769ms step_avg:151.87ms
step:128/1480 train_time:17915ms step_avg:151.82ms
step:129/1480 train_time:18059ms step_avg:151.76ms
step:130/1480 train_time:18205ms step_avg:151.71ms
step:131/1480 train_time:18351ms step_avg:151.66ms
step:132/1480 train_time:18496ms step_avg:151.60ms
step:133/1480 train_time:18641ms step_avg:151.55ms
step:134/1480 train_time:18788ms step_avg:151.51ms
step:135/1480 train_time:18934ms step_avg:151.47ms
step:136/1480 train_time:19078ms step_avg:151.42ms
step:137/1480 train_time:19223ms step_avg:151.36ms
step:138/1480 train_time:19369ms step_avg:151.32ms
step:139/1480 train_time:19515ms step_avg:151.28ms
step:140/1480 train_time:19660ms step_avg:151.23ms
step:141/1480 train_time:19806ms step_avg:151.19ms
step:142/1480 train_time:19953ms step_avg:151.16ms
step:143/1480 train_time:20098ms step_avg:151.11ms
step:144/1480 train_time:20242ms step_avg:151.06ms
step:145/1480 train_time:20389ms step_avg:151.03ms
step:146/1480 train_time:20535ms step_avg:151.00ms
step:147/1480 train_time:20681ms step_avg:150.96ms
step:148/1480 train_time:20828ms step_avg:150.93ms
step:149/1480 train_time:20974ms step_avg:150.89ms
step:150/1480 train_time:21120ms step_avg:150.85ms
step:151/1480 train_time:21265ms step_avg:150.82ms
step:152/1480 train_time:21412ms step_avg:150.79ms
step:153/1480 train_time:21557ms step_avg:150.75ms
step:154/1480 train_time:21702ms step_avg:150.71ms
step:155/1480 train_time:21849ms step_avg:150.68ms
step:156/1480 train_time:21995ms step_avg:150.65ms
step:157/1480 train_time:22140ms step_avg:150.61ms
step:158/1480 train_time:22286ms step_avg:150.58ms
step:159/1480 train_time:22433ms step_avg:150.56ms
step:160/1480 train_time:22578ms step_avg:150.52ms
step:161/1480 train_time:22724ms step_avg:150.49ms
step:162/1480 train_time:22871ms step_avg:150.47ms
step:163/1480 train_time:23016ms step_avg:150.43ms
step:164/1480 train_time:23163ms step_avg:150.41ms
step:165/1480 train_time:23310ms step_avg:150.39ms
step:166/1480 train_time:23456ms step_avg:150.36ms
step:167/1480 train_time:23601ms step_avg:150.32ms
step:168/1480 train_time:23747ms step_avg:150.30ms
step:169/1480 train_time:23893ms step_avg:150.27ms
step:170/1480 train_time:24038ms step_avg:150.24ms
step:171/1480 train_time:24183ms step_avg:150.20ms
step:172/1480 train_time:24330ms step_avg:150.18ms
step:173/1480 train_time:24475ms step_avg:150.15ms
step:174/1480 train_time:24621ms step_avg:150.13ms
step:175/1480 train_time:24766ms step_avg:150.10ms
step:176/1480 train_time:24913ms step_avg:150.08ms
step:177/1480 train_time:25057ms step_avg:150.04ms
step:178/1480 train_time:25205ms step_avg:150.03ms
step:179/1480 train_time:25352ms step_avg:150.01ms
step:180/1480 train_time:25497ms step_avg:149.98ms
step:181/1480 train_time:25641ms step_avg:149.95ms
step:182/1480 train_time:25787ms step_avg:149.93ms
step:183/1480 train_time:26319ms step_avg:152.13ms
step:184/1480 train_time:26424ms step_avg:151.86ms
step:185/1480 train_time:26570ms step_avg:151.83ms
step:186/1480 train_time:26715ms step_avg:151.79ms
step:187/1480 train_time:26860ms step_avg:151.75ms
step:188/1480 train_time:27005ms step_avg:151.71ms
step:189/1480 train_time:27185ms step_avg:151.87ms
step:190/1480 train_time:27296ms step_avg:151.64ms
step:191/1480 train_time:27441ms step_avg:151.61ms
step:192/1480 train_time:27586ms step_avg:151.57ms
step:193/1480 train_time:27733ms step_avg:151.55ms
step:194/1480 train_time:27877ms step_avg:151.51ms
step:195/1480 train_time:28023ms step_avg:151.48ms
step:196/1480 train_time:28170ms step_avg:151.45ms
step:197/1480 train_time:28315ms step_avg:151.42ms
step:198/1480 train_time:28460ms step_avg:151.38ms
step:199/1480 train_time:28607ms step_avg:151.36ms
step:200/1480 train_time:28755ms step_avg:151.34ms
step:201/1480 train_time:28901ms step_avg:151.31ms
step:202/1480 train_time:29044ms step_avg:151.27ms
step:203/1480 train_time:29190ms step_avg:151.25ms
step:204/1480 train_time:29336ms step_avg:151.22ms
step:205/1480 train_time:29480ms step_avg:151.18ms
step:206/1480 train_time:29626ms step_avg:151.15ms
step:207/1480 train_time:29772ms step_avg:151.13ms
step:208/1480 train_time:29918ms step_avg:151.10ms
step:209/1480 train_time:30063ms step_avg:151.07ms
step:210/1480 train_time:30208ms step_avg:151.04ms
step:211/1480 train_time:30354ms step_avg:151.02ms
step:212/1480 train_time:30499ms step_avg:150.99ms
step:213/1480 train_time:30644ms step_avg:150.96ms
step:214/1480 train_time:30791ms step_avg:150.94ms
step:215/1480 train_time:30937ms step_avg:150.91ms
step:216/1480 train_time:31082ms step_avg:150.88ms
step:217/1480 train_time:31228ms step_avg:150.86ms
step:218/1480 train_time:31374ms step_avg:150.84ms
step:219/1480 train_time:31520ms step_avg:150.81ms
step:220/1480 train_time:31665ms step_avg:150.79ms
step:221/1480 train_time:32208ms step_avg:152.65ms
step:222/1480 train_time:32316ms step_avg:152.43ms
step:223/1480 train_time:32464ms step_avg:152.41ms
step:224/1480 train_time:32613ms step_avg:152.40ms
step:225/1480 train_time:32761ms step_avg:152.38ms
step:226/1480 train_time:32910ms step_avg:152.36ms
step:227/1480 train_time:33058ms step_avg:152.34ms
step:228/1480 train_time:33208ms step_avg:152.33ms
step:229/1480 train_time:33357ms step_avg:152.31ms
step:230/1480 train_time:33506ms step_avg:152.30ms
step:231/1480 train_time:33655ms step_avg:152.28ms
step:232/1480 train_time:33802ms step_avg:152.26ms
step:233/1480 train_time:33952ms step_avg:152.25ms
step:234/1480 train_time:34099ms step_avg:152.23ms
step:235/1480 train_time:34247ms step_avg:152.21ms
step:236/1480 train_time:34397ms step_avg:152.20ms
step:237/1480 train_time:34544ms step_avg:152.18ms
step:238/1480 train_time:34693ms step_avg:152.16ms
step:239/1480 train_time:34841ms step_avg:152.14ms
step:240/1480 train_time:34989ms step_avg:152.13ms
step:241/1480 train_time:35138ms step_avg:152.11ms
step:242/1480 train_time:35286ms step_avg:152.09ms
step:243/1480 train_time:35436ms step_avg:152.09ms
step:244/1480 train_time:35583ms step_avg:152.07ms
step:245/1480 train_time:35733ms step_avg:152.06ms
step:246/1480 train_time:35881ms step_avg:152.04ms
step:247/1480 train_time:36030ms step_avg:152.03ms
step:248/1480 train_time:36178ms step_avg:152.01ms
step:249/1480 train_time:36327ms step_avg:152.00ms
step:250/1480 train_time:36476ms step_avg:151.98ms
step:250/1480 val_loss:3.9917 train_time:36544ms step_avg:152.26ms
step:251/1480 train_time:36640ms step_avg:152.03ms
step:252/1480 train_time:36784ms step_avg:152.00ms
step:253/1480 train_time:36933ms step_avg:151.99ms
step:254/1480 train_time:37082ms step_avg:151.98ms
step:255/1480 train_time:37229ms step_avg:151.96ms
step:256/1480 train_time:37378ms step_avg:151.94ms
step:257/1480 train_time:37525ms step_avg:151.92ms
step:258/1480 train_time:37673ms step_avg:151.91ms
step:259/1480 train_time:37822ms step_avg:151.90ms
step:260/1480 train_time:37971ms step_avg:151.88ms
step:261/1480 train_time:38120ms step_avg:151.87ms
step:262/1480 train_time:38268ms step_avg:151.86ms
step:263/1480 train_time:38416ms step_avg:151.84ms
step:264/1480 train_time:38564ms step_avg:151.83ms
step:265/1480 train_time:38712ms step_avg:151.81ms
step:266/1480 train_time:38861ms step_avg:151.80ms
step:267/1480 train_time:39008ms step_avg:151.78ms
step:268/1480 train_time:39158ms step_avg:151.77ms
step:269/1480 train_time:39305ms step_avg:151.76ms
step:270/1480 train_time:39454ms step_avg:151.75ms
step:271/1480 train_time:39603ms step_avg:151.74ms
step:272/1480 train_time:39752ms step_avg:151.72ms
step:273/1480 train_time:39900ms step_avg:151.71ms
step:274/1480 train_time:40049ms step_avg:151.70ms
step:275/1480 train_time:40198ms step_avg:151.69ms
step:276/1480 train_time:40346ms step_avg:151.68ms
step:277/1480 train_time:40495ms step_avg:151.67ms
step:278/1480 train_time:40643ms step_avg:151.65ms
step:279/1480 train_time:40791ms step_avg:151.64ms
step:280/1480 train_time:40941ms step_avg:151.63ms
step:281/1480 train_time:41089ms step_avg:151.62ms
step:282/1480 train_time:41237ms step_avg:151.61ms
step:283/1480 train_time:41385ms step_avg:151.60ms
step:284/1480 train_time:41534ms step_avg:151.58ms
step:285/1480 train_time:41683ms step_avg:151.57ms
step:286/1480 train_time:41831ms step_avg:151.56ms
step:287/1480 train_time:41980ms step_avg:151.55ms
step:288/1480 train_time:42130ms step_avg:151.54ms
step:289/1480 train_time:42279ms step_avg:151.54ms
step:290/1480 train_time:42426ms step_avg:151.52ms
step:291/1480 train_time:42575ms step_avg:151.51ms
step:292/1480 train_time:42723ms step_avg:151.50ms
step:293/1480 train_time:42871ms step_avg:151.49ms
step:294/1480 train_time:43020ms step_avg:151.48ms
step:295/1480 train_time:43168ms step_avg:151.47ms
step:296/1480 train_time:43318ms step_avg:151.46ms
step:297/1480 train_time:43466ms step_avg:151.45ms
step:298/1480 train_time:43615ms step_avg:151.44ms
step:299/1480 train_time:43763ms step_avg:151.43ms
step:300/1480 train_time:43911ms step_avg:151.42ms
step:301/1480 train_time:44060ms step_avg:151.41ms
step:302/1480 train_time:44207ms step_avg:151.40ms
step:303/1480 train_time:44356ms step_avg:151.39ms
step:304/1480 train_time:44504ms step_avg:151.37ms
step:305/1480 train_time:44653ms step_avg:151.37ms
step:306/1480 train_time:44801ms step_avg:151.36ms
step:307/1480 train_time:44951ms step_avg:151.35ms
step:308/1480 train_time:45100ms step_avg:151.34ms
step:309/1480 train_time:45248ms step_avg:151.33ms
step:310/1480 train_time:45397ms step_avg:151.32ms
step:311/1480 train_time:45545ms step_avg:151.31ms
step:312/1480 train_time:45693ms step_avg:151.30ms
step:313/1480 train_time:45842ms step_avg:151.30ms
step:314/1480 train_time:45991ms step_avg:151.29ms
step:315/1480 train_time:46141ms step_avg:151.28ms
step:316/1480 train_time:46289ms step_avg:151.27ms
step:317/1480 train_time:46438ms step_avg:151.26ms
step:318/1480 train_time:46586ms step_avg:151.25ms
step:319/1480 train_time:46735ms step_avg:151.24ms
step:320/1480 train_time:46884ms step_avg:151.24ms
step:321/1480 train_time:47032ms step_avg:151.23ms
step:322/1480 train_time:47181ms step_avg:151.22ms
step:323/1480 train_time:47329ms step_avg:151.21ms
step:324/1480 train_time:47480ms step_avg:151.21ms
step:325/1480 train_time:47627ms step_avg:151.20ms
step:326/1480 train_time:47775ms step_avg:151.19ms
step:327/1480 train_time:47923ms step_avg:151.18ms
step:328/1480 train_time:48071ms step_avg:151.17ms
step:329/1480 train_time:48219ms step_avg:151.16ms
step:330/1480 train_time:48368ms step_avg:151.15ms
step:331/1480 train_time:48520ms step_avg:151.15ms
step:332/1480 train_time:48670ms step_avg:151.15ms
step:333/1480 train_time:48821ms step_avg:151.15ms
step:334/1480 train_time:48971ms step_avg:151.15ms
step:335/1480 train_time:49122ms step_avg:151.15ms
step:336/1480 train_time:49272ms step_avg:151.14ms
step:337/1480 train_time:49423ms step_avg:151.14ms
step:338/1480 train_time:49573ms step_avg:151.14ms
step:339/1480 train_time:49724ms step_avg:151.14ms
step:340/1480 train_time:49874ms step_avg:151.13ms
step:341/1480 train_time:50024ms step_avg:151.13ms
step:342/1480 train_time:50175ms step_avg:151.13ms
step:343/1480 train_time:50325ms step_avg:151.13ms
step:344/1480 train_time:50476ms step_avg:151.13ms
step:345/1480 train_time:50626ms step_avg:151.12ms
step:346/1480 train_time:50777ms step_avg:151.12ms
step:347/1480 train_time:50927ms step_avg:151.12ms
step:348/1480 train_time:51079ms step_avg:151.12ms
step:349/1480 train_time:51229ms step_avg:151.12ms
step:350/1480 train_time:51381ms step_avg:151.12ms
step:351/1480 train_time:51531ms step_avg:151.12ms
step:352/1480 train_time:51682ms step_avg:151.12ms
step:353/1480 train_time:51833ms step_avg:151.12ms
step:354/1480 train_time:51984ms step_avg:151.12ms
step:355/1480 train_time:52135ms step_avg:151.12ms
step:356/1480 train_time:52287ms step_avg:151.12ms
step:357/1480 train_time:52438ms step_avg:151.12ms
step:358/1480 train_time:52588ms step_avg:151.11ms
step:359/1480 train_time:52739ms step_avg:151.11ms
step:360/1480 train_time:52890ms step_avg:151.11ms
step:361/1480 train_time:53041ms step_avg:151.12ms
step:362/1480 train_time:53193ms step_avg:151.12ms
step:363/1480 train_time:53343ms step_avg:151.11ms
step:364/1480 train_time:53496ms step_avg:151.12ms
step:365/1480 train_time:53646ms step_avg:151.11ms
step:366/1480 train_time:53796ms step_avg:151.11ms
step:367/1480 train_time:53946ms step_avg:151.11ms
step:368/1480 train_time:54096ms step_avg:151.11ms
step:369/1480 train_time:54247ms step_avg:151.11ms
step:370/1480 train_time:54398ms step_avg:151.11ms
step:371/1480 train_time:54549ms step_avg:151.11ms
step:372/1480 train_time:54700ms step_avg:151.11ms
step:373/1480 train_time:54851ms step_avg:151.10ms
step:374/1480 train_time:55000ms step_avg:151.10ms
step:375/1480 train_time:55152ms step_avg:151.10ms
step:375/1480 val_loss:3.8083 train_time:55220ms step_avg:151.29ms
step:376/1480 train_time:55312ms step_avg:151.13ms
step:377/1480 train_time:55461ms step_avg:151.12ms
step:378/1480 train_time:55613ms step_avg:151.12ms
step:379/1480 train_time:55783ms step_avg:151.17ms
step:380/1480 train_time:55913ms step_avg:151.12ms
step:381/1480 train_time:56062ms step_avg:151.11ms
step:382/1480 train_time:56213ms step_avg:151.11ms
step:383/1480 train_time:56365ms step_avg:151.11ms
step:384/1480 train_time:56518ms step_avg:151.12ms
step:385/1480 train_time:56669ms step_avg:151.12ms
step:386/1480 train_time:56819ms step_avg:151.11ms
step:387/1480 train_time:56970ms step_avg:151.11ms
step:388/1480 train_time:57120ms step_avg:151.11ms
step:389/1480 train_time:57271ms step_avg:151.11ms
step:390/1480 train_time:57421ms step_avg:151.11ms
step:391/1480 train_time:57572ms step_avg:151.11ms
step:392/1480 train_time:57723ms step_avg:151.11ms
step:393/1480 train_time:57875ms step_avg:151.11ms
step:394/1480 train_time:58025ms step_avg:151.11ms
step:395/1480 train_time:58175ms step_avg:151.10ms
step:396/1480 train_time:58326ms step_avg:151.10ms
step:397/1480 train_time:58477ms step_avg:151.10ms
step:398/1480 train_time:58628ms step_avg:151.10ms
step:399/1480 train_time:58778ms step_avg:151.10ms
step:400/1480 train_time:58930ms step_avg:151.10ms
step:401/1480 train_time:59080ms step_avg:151.10ms
step:402/1480 train_time:59232ms step_avg:151.10ms
step:403/1480 train_time:59383ms step_avg:151.10ms
step:404/1480 train_time:59535ms step_avg:151.10ms
step:405/1480 train_time:59684ms step_avg:151.10ms
step:406/1480 train_time:59836ms step_avg:151.10ms
step:407/1480 train_time:59987ms step_avg:151.10ms
step:408/1480 train_time:60138ms step_avg:151.10ms
step:409/1480 train_time:60288ms step_avg:151.10ms
step:410/1480 train_time:60439ms step_avg:151.10ms
step:411/1480 train_time:60590ms step_avg:151.10ms
step:412/1480 train_time:60740ms step_avg:151.10ms
step:413/1480 train_time:60892ms step_avg:151.10ms
step:414/1480 train_time:61042ms step_avg:151.09ms
step:415/1480 train_time:61195ms step_avg:151.10ms
step:416/1480 train_time:61346ms step_avg:151.10ms
step:417/1480 train_time:61497ms step_avg:151.10ms
step:418/1480 train_time:61648ms step_avg:151.10ms
step:419/1480 train_time:61799ms step_avg:151.10ms
step:420/1480 train_time:61950ms step_avg:151.10ms
step:421/1480 train_time:62100ms step_avg:151.09ms
step:422/1480 train_time:62251ms step_avg:151.10ms
step:423/1480 train_time:62402ms step_avg:151.10ms
step:424/1480 train_time:62554ms step_avg:151.10ms
step:425/1480 train_time:62704ms step_avg:151.09ms
step:426/1480 train_time:62855ms step_avg:151.09ms
step:427/1480 train_time:63006ms step_avg:151.09ms
step:428/1480 train_time:63157ms step_avg:151.09ms
step:429/1480 train_time:63308ms step_avg:151.09ms
step:430/1480 train_time:63459ms step_avg:151.09ms
step:431/1480 train_time:63609ms step_avg:151.09ms
step:432/1480 train_time:63760ms step_avg:151.09ms
step:433/1480 train_time:63910ms step_avg:151.09ms
step:434/1480 train_time:64061ms step_avg:151.09ms
step:435/1480 train_time:64212ms step_avg:151.09ms
step:436/1480 train_time:64363ms step_avg:151.09ms
step:437/1480 train_time:64514ms step_avg:151.09ms
step:438/1480 train_time:64666ms step_avg:151.09ms
step:439/1480 train_time:64818ms step_avg:151.09ms
step:440/1480 train_time:64969ms step_avg:151.09ms
step:441/1480 train_time:65121ms step_avg:151.09ms
step:442/1480 train_time:65275ms step_avg:151.10ms
step:443/1480 train_time:65428ms step_avg:151.10ms
step:444/1480 train_time:65580ms step_avg:151.11ms
step:445/1480 train_time:65733ms step_avg:151.11ms
step:446/1480 train_time:65886ms step_avg:151.11ms
step:447/1480 train_time:66038ms step_avg:151.12ms
step:448/1480 train_time:66193ms step_avg:151.13ms
step:449/1480 train_time:66346ms step_avg:151.13ms
step:450/1480 train_time:66499ms step_avg:151.13ms
step:451/1480 train_time:66653ms step_avg:151.14ms
step:452/1480 train_time:66806ms step_avg:151.14ms
step:453/1480 train_time:66958ms step_avg:151.15ms
step:454/1480 train_time:67110ms step_avg:151.15ms
step:455/1480 train_time:67263ms step_avg:151.15ms
step:456/1480 train_time:67416ms step_avg:151.16ms
step:457/1480 train_time:67570ms step_avg:151.16ms
step:458/1480 train_time:67722ms step_avg:151.17ms
step:459/1480 train_time:67876ms step_avg:151.17ms
step:460/1480 train_time:68028ms step_avg:151.17ms
step:461/1480 train_time:68180ms step_avg:151.17ms
step:462/1480 train_time:68334ms step_avg:151.18ms
step:463/1480 train_time:68486ms step_avg:151.18ms
step:464/1480 train_time:68639ms step_avg:151.19ms
step:465/1480 train_time:68792ms step_avg:151.19ms
step:466/1480 train_time:68945ms step_avg:151.20ms
step:467/1480 train_time:69098ms step_avg:151.20ms
step:468/1480 train_time:69250ms step_avg:151.20ms
step:469/1480 train_time:69402ms step_avg:151.20ms
step:470/1480 train_time:69555ms step_avg:151.21ms
step:471/1480 train_time:69707ms step_avg:151.21ms
step:472/1480 train_time:69859ms step_avg:151.21ms
step:473/1480 train_time:70013ms step_avg:151.22ms
step:474/1480 train_time:70167ms step_avg:151.22ms
step:475/1480 train_time:70319ms step_avg:151.22ms
step:476/1480 train_time:70473ms step_avg:151.23ms
step:477/1480 train_time:70626ms step_avg:151.23ms
step:478/1480 train_time:70779ms step_avg:151.24ms
step:479/1480 train_time:70932ms step_avg:151.24ms
step:480/1480 train_time:71084ms step_avg:151.24ms
step:481/1480 train_time:71237ms step_avg:151.25ms
step:482/1480 train_time:71390ms step_avg:151.25ms
step:483/1480 train_time:71541ms step_avg:151.25ms
step:484/1480 train_time:71696ms step_avg:151.26ms
step:485/1480 train_time:71849ms step_avg:151.26ms
step:486/1480 train_time:72001ms step_avg:151.26ms
step:487/1480 train_time:72155ms step_avg:151.27ms
step:488/1480 train_time:72307ms step_avg:151.27ms
step:489/1480 train_time:72460ms step_avg:151.27ms
step:490/1480 train_time:72613ms step_avg:151.28ms
step:491/1480 train_time:72768ms step_avg:151.29ms
step:492/1480 train_time:72921ms step_avg:151.29ms
step:493/1480 train_time:73074ms step_avg:151.29ms
step:494/1480 train_time:73227ms step_avg:151.30ms
step:495/1480 train_time:73380ms step_avg:151.30ms
step:496/1480 train_time:73535ms step_avg:151.31ms
step:497/1480 train_time:73687ms step_avg:151.31ms
step:498/1480 train_time:73839ms step_avg:151.31ms
step:499/1480 train_time:73993ms step_avg:151.31ms
step:500/1480 train_time:74146ms step_avg:151.32ms
step:500/1480 val_loss:3.6829 train_time:74216ms step_avg:151.46ms
step:501/1480 train_time:74307ms step_avg:151.34ms
step:502/1480 train_time:74459ms step_avg:151.34ms
step:503/1480 train_time:74612ms step_avg:151.34ms
step:504/1480 train_time:74764ms step_avg:151.34ms
step:505/1480 train_time:74916ms step_avg:151.35ms
step:506/1480 train_time:75068ms step_avg:151.35ms
step:507/1480 train_time:75220ms step_avg:151.35ms
step:508/1480 train_time:75376ms step_avg:151.36ms
step:509/1480 train_time:75529ms step_avg:151.36ms
step:510/1480 train_time:75683ms step_avg:151.37ms
step:511/1480 train_time:75836ms step_avg:151.37ms
step:512/1480 train_time:75988ms step_avg:151.37ms
step:513/1480 train_time:76141ms step_avg:151.37ms
step:514/1480 train_time:76296ms step_avg:151.38ms
step:515/1480 train_time:76449ms step_avg:151.38ms
step:516/1480 train_time:76603ms step_avg:151.39ms
step:517/1480 train_time:76756ms step_avg:151.39ms
step:518/1480 train_time:76908ms step_avg:151.39ms
step:519/1480 train_time:77062ms step_avg:151.40ms
step:520/1480 train_time:77215ms step_avg:151.40ms
step:521/1480 train_time:77367ms step_avg:151.40ms
step:522/1480 train_time:77521ms step_avg:151.41ms
step:523/1480 train_time:77673ms step_avg:151.41ms
step:524/1480 train_time:77826ms step_avg:151.41ms
step:525/1480 train_time:77979ms step_avg:151.42ms
step:526/1480 train_time:78132ms step_avg:151.42ms
step:527/1480 train_time:78285ms step_avg:151.42ms
step:528/1480 train_time:78437ms step_avg:151.42ms
step:529/1480 train_time:78589ms step_avg:151.42ms
step:530/1480 train_time:78742ms step_avg:151.43ms
step:531/1480 train_time:78895ms step_avg:151.43ms
step:532/1480 train_time:79047ms step_avg:151.43ms
step:533/1480 train_time:79199ms step_avg:151.43ms
step:534/1480 train_time:79352ms step_avg:151.44ms
step:535/1480 train_time:79505ms step_avg:151.44ms
step:536/1480 train_time:79658ms step_avg:151.44ms
step:537/1480 train_time:79811ms step_avg:151.44ms
step:538/1480 train_time:79965ms step_avg:151.45ms
step:539/1480 train_time:80118ms step_avg:151.45ms
step:540/1480 train_time:80271ms step_avg:151.46ms
step:541/1480 train_time:80424ms step_avg:151.46ms
step:542/1480 train_time:80577ms step_avg:151.46ms
step:543/1480 train_time:80729ms step_avg:151.46ms
step:544/1480 train_time:80883ms step_avg:151.47ms
step:545/1480 train_time:81036ms step_avg:151.47ms
step:546/1480 train_time:81189ms step_avg:151.47ms
step:547/1480 train_time:81342ms step_avg:151.47ms
step:548/1480 train_time:81496ms step_avg:151.48ms
step:549/1480 train_time:81648ms step_avg:151.48ms
step:550/1480 train_time:81802ms step_avg:151.48ms
step:551/1480 train_time:81957ms step_avg:151.49ms
step:552/1480 train_time:82112ms step_avg:151.50ms
step:553/1480 train_time:82268ms step_avg:151.51ms
step:554/1480 train_time:82423ms step_avg:151.51ms
step:555/1480 train_time:82577ms step_avg:151.52ms
step:556/1480 train_time:82732ms step_avg:151.52ms
step:557/1480 train_time:82886ms step_avg:151.53ms
step:558/1480 train_time:83042ms step_avg:151.54ms
step:559/1480 train_time:83197ms step_avg:151.54ms
step:560/1480 train_time:83351ms step_avg:151.55ms
step:561/1480 train_time:83506ms step_avg:151.55ms
step:562/1480 train_time:83661ms step_avg:151.56ms
step:563/1480 train_time:83816ms step_avg:151.57ms
step:564/1480 train_time:83970ms step_avg:151.57ms
step:565/1480 train_time:84124ms step_avg:151.57ms
step:566/1480 train_time:84280ms step_avg:151.58ms
step:567/1480 train_time:84435ms step_avg:151.59ms
step:568/1480 train_time:84590ms step_avg:151.59ms
step:569/1480 train_time:84764ms step_avg:151.63ms
step:570/1480 train_time:84898ms step_avg:151.60ms
step:571/1480 train_time:85052ms step_avg:151.61ms
step:572/1480 train_time:85206ms step_avg:151.61ms
step:573/1480 train_time:85361ms step_avg:151.62ms
step:574/1480 train_time:85519ms step_avg:151.63ms
step:575/1480 train_time:85674ms step_avg:151.63ms
step:576/1480 train_time:85829ms step_avg:151.64ms
step:577/1480 train_time:85983ms step_avg:151.65ms
step:578/1480 train_time:86138ms step_avg:151.65ms
step:579/1480 train_time:86293ms step_avg:151.66ms
step:580/1480 train_time:86448ms step_avg:151.66ms
step:581/1480 train_time:86602ms step_avg:151.67ms
step:582/1480 train_time:86758ms step_avg:151.67ms
step:583/1480 train_time:86913ms step_avg:151.68ms
step:584/1480 train_time:87069ms step_avg:151.69ms
step:585/1480 train_time:87223ms step_avg:151.69ms
step:586/1480 train_time:87378ms step_avg:151.70ms
step:587/1480 train_time:87534ms step_avg:151.71ms
step:588/1480 train_time:87688ms step_avg:151.71ms
step:589/1480 train_time:87843ms step_avg:151.71ms
step:590/1480 train_time:87998ms step_avg:151.72ms
step:591/1480 train_time:88153ms step_avg:151.73ms
step:592/1480 train_time:88308ms step_avg:151.73ms
step:593/1480 train_time:88463ms step_avg:151.74ms
step:594/1480 train_time:88617ms step_avg:151.74ms
step:595/1480 train_time:88773ms step_avg:151.75ms
step:596/1480 train_time:88929ms step_avg:151.76ms
step:597/1480 train_time:89084ms step_avg:151.76ms
step:598/1480 train_time:89238ms step_avg:151.77ms
step:599/1480 train_time:89393ms step_avg:151.77ms
step:600/1480 train_time:89547ms step_avg:151.77ms
step:601/1480 train_time:89701ms step_avg:151.78ms
step:602/1480 train_time:89856ms step_avg:151.78ms
step:603/1480 train_time:90011ms step_avg:151.79ms
step:604/1480 train_time:90165ms step_avg:151.79ms
step:605/1480 train_time:90321ms step_avg:151.80ms
step:606/1480 train_time:90475ms step_avg:151.80ms
step:607/1480 train_time:90631ms step_avg:151.81ms
step:608/1480 train_time:90786ms step_avg:151.82ms
step:609/1480 train_time:90941ms step_avg:151.82ms
step:610/1480 train_time:91095ms step_avg:151.83ms
step:611/1480 train_time:91250ms step_avg:151.83ms
step:612/1480 train_time:91405ms step_avg:151.84ms
step:613/1480 train_time:91560ms step_avg:151.84ms
step:614/1480 train_time:91715ms step_avg:151.85ms
step:615/1480 train_time:91870ms step_avg:151.85ms
step:616/1480 train_time:92024ms step_avg:151.85ms
step:617/1480 train_time:92179ms step_avg:151.86ms
step:618/1480 train_time:92333ms step_avg:151.86ms
step:619/1480 train_time:92490ms step_avg:151.87ms
step:620/1480 train_time:92645ms step_avg:151.88ms
step:621/1480 train_time:92799ms step_avg:151.88ms
step:622/1480 train_time:92955ms step_avg:151.89ms
step:623/1480 train_time:93111ms step_avg:151.89ms
step:624/1480 train_time:93266ms step_avg:151.90ms
step:625/1480 train_time:93421ms step_avg:151.90ms
step:625/1480 val_loss:3.6026 train_time:93491ms step_avg:152.02ms
step:626/1480 train_time:93586ms step_avg:151.93ms
step:627/1480 train_time:93737ms step_avg:151.92ms
step:628/1480 train_time:93892ms step_avg:151.93ms
step:629/1480 train_time:94046ms step_avg:151.93ms
step:630/1480 train_time:94202ms step_avg:151.94ms
step:631/1480 train_time:94355ms step_avg:151.94ms
step:632/1480 train_time:94509ms step_avg:151.94ms
step:633/1480 train_time:94664ms step_avg:151.95ms
step:634/1480 train_time:94819ms step_avg:151.95ms
step:635/1480 train_time:94974ms step_avg:151.96ms
step:636/1480 train_time:95129ms step_avg:151.96ms
step:637/1480 train_time:95283ms step_avg:151.97ms
step:638/1480 train_time:95438ms step_avg:151.97ms
step:639/1480 train_time:95592ms step_avg:151.98ms
step:640/1480 train_time:95747ms step_avg:151.98ms
step:641/1480 train_time:95901ms step_avg:151.98ms
step:642/1480 train_time:96055ms step_avg:151.99ms
step:643/1480 train_time:96210ms step_avg:151.99ms
step:644/1480 train_time:96364ms step_avg:151.99ms
step:645/1480 train_time:96519ms step_avg:152.00ms
step:646/1480 train_time:96674ms step_avg:152.00ms
step:647/1480 train_time:96829ms step_avg:152.01ms
step:648/1480 train_time:96985ms step_avg:152.01ms
step:649/1480 train_time:97140ms step_avg:152.02ms
step:650/1480 train_time:97295ms step_avg:152.02ms
step:651/1480 train_time:97449ms step_avg:152.03ms
step:652/1480 train_time:97604ms step_avg:152.03ms
step:653/1480 train_time:97758ms step_avg:152.03ms
step:654/1480 train_time:97914ms step_avg:152.04ms
step:655/1480 train_time:98068ms step_avg:152.04ms
step:656/1480 train_time:98223ms step_avg:152.05ms
step:657/1480 train_time:98378ms step_avg:152.05ms
step:658/1480 train_time:98533ms step_avg:152.06ms
step:659/1480 train_time:98688ms step_avg:152.06ms
step:660/1480 train_time:98846ms step_avg:152.07ms
step:661/1480 train_time:99003ms step_avg:152.08ms
step:662/1480 train_time:99159ms step_avg:152.08ms
step:663/1480 train_time:99314ms step_avg:152.09ms
step:664/1480 train_time:99469ms step_avg:152.09ms
step:665/1480 train_time:99626ms step_avg:152.10ms
step:666/1480 train_time:99782ms step_avg:152.11ms
step:667/1480 train_time:99940ms step_avg:152.12ms
step:668/1480 train_time:100096ms step_avg:152.12ms
step:669/1480 train_time:100254ms step_avg:152.13ms
step:670/1480 train_time:100409ms step_avg:152.13ms
step:671/1480 train_time:100566ms step_avg:152.14ms
step:672/1480 train_time:100723ms step_avg:152.15ms
step:673/1480 train_time:100879ms step_avg:152.16ms
step:674/1480 train_time:101036ms step_avg:152.16ms
step:675/1480 train_time:101192ms step_avg:152.17ms
step:676/1480 train_time:101349ms step_avg:152.18ms
step:677/1480 train_time:101506ms step_avg:152.18ms
step:678/1480 train_time:101661ms step_avg:152.19ms
step:679/1480 train_time:101819ms step_avg:152.20ms
step:680/1480 train_time:101977ms step_avg:152.20ms
step:681/1480 train_time:102132ms step_avg:152.21ms
step:682/1480 train_time:102288ms step_avg:152.21ms
step:683/1480 train_time:102445ms step_avg:152.22ms
step:684/1480 train_time:102602ms step_avg:152.23ms
step:685/1480 train_time:102757ms step_avg:152.23ms
step:686/1480 train_time:102914ms step_avg:152.24ms
step:687/1480 train_time:103069ms step_avg:152.24ms
step:688/1480 train_time:103227ms step_avg:152.25ms
step:689/1480 train_time:103384ms step_avg:152.26ms
step:690/1480 train_time:103542ms step_avg:152.27ms
step:691/1480 train_time:103699ms step_avg:152.27ms
step:692/1480 train_time:103854ms step_avg:152.28ms
step:693/1480 train_time:104010ms step_avg:152.28ms
step:694/1480 train_time:104168ms step_avg:152.29ms
step:695/1480 train_time:104323ms step_avg:152.30ms
step:696/1480 train_time:104479ms step_avg:152.30ms
step:697/1480 train_time:104636ms step_avg:152.31ms
step:698/1480 train_time:104791ms step_avg:152.31ms
step:699/1480 train_time:104948ms step_avg:152.32ms
step:700/1480 train_time:105104ms step_avg:152.33ms
step:701/1480 train_time:105259ms step_avg:152.33ms
step:702/1480 train_time:105416ms step_avg:152.34ms
step:703/1480 train_time:105573ms step_avg:152.34ms
step:704/1480 train_time:105728ms step_avg:152.35ms
step:705/1480 train_time:105885ms step_avg:152.35ms
step:706/1480 train_time:106045ms step_avg:152.36ms
step:707/1480 train_time:106201ms step_avg:152.37ms
step:708/1480 train_time:106358ms step_avg:152.38ms
step:709/1480 train_time:106512ms step_avg:152.38ms
step:710/1480 train_time:106668ms step_avg:152.38ms
step:711/1480 train_time:106825ms step_avg:152.39ms
step:712/1480 train_time:106982ms step_avg:152.40ms
step:713/1480 train_time:107141ms step_avg:152.41ms
step:714/1480 train_time:107299ms step_avg:152.41ms
step:715/1480 train_time:107454ms step_avg:152.42ms
step:716/1480 train_time:107609ms step_avg:152.42ms
step:717/1480 train_time:107766ms step_avg:152.43ms
step:718/1480 train_time:107921ms step_avg:152.43ms
step:719/1480 train_time:108078ms step_avg:152.44ms
step:720/1480 train_time:108236ms step_avg:152.45ms
step:721/1480 train_time:108393ms step_avg:152.45ms
step:722/1480 train_time:108549ms step_avg:152.46ms
step:723/1480 train_time:108705ms step_avg:152.46ms
step:724/1480 train_time:108860ms step_avg:152.47ms
step:725/1480 train_time:109017ms step_avg:152.47ms
step:726/1480 train_time:109174ms step_avg:152.48ms
step:727/1480 train_time:109331ms step_avg:152.48ms
step:728/1480 train_time:109486ms step_avg:152.49ms
step:729/1480 train_time:109645ms step_avg:152.50ms
step:730/1480 train_time:109802ms step_avg:152.50ms
step:731/1480 train_time:109958ms step_avg:152.51ms
step:732/1480 train_time:110113ms step_avg:152.51ms
step:733/1480 train_time:110269ms step_avg:152.52ms
step:734/1480 train_time:110427ms step_avg:152.52ms
step:735/1480 train_time:110583ms step_avg:152.53ms
step:736/1480 train_time:110740ms step_avg:152.53ms
step:737/1480 train_time:110896ms step_avg:152.54ms
step:738/1480 train_time:111051ms step_avg:152.54ms
step:739/1480 train_time:111207ms step_avg:152.55ms
step:740/1480 train_time:111366ms step_avg:152.56ms
step:741/1480 train_time:111524ms step_avg:152.56ms
step:742/1480 train_time:111681ms step_avg:152.57ms
step:743/1480 train_time:111838ms step_avg:152.58ms
step:744/1480 train_time:111994ms step_avg:152.58ms
step:745/1480 train_time:112151ms step_avg:152.59ms
step:746/1480 train_time:112307ms step_avg:152.59ms
step:747/1480 train_time:112462ms step_avg:152.59ms
step:748/1480 train_time:112622ms step_avg:152.60ms
step:749/1480 train_time:112780ms step_avg:152.61ms
step:750/1480 train_time:112937ms step_avg:152.62ms
step:750/1480 val_loss:3.5468 train_time:113008ms step_avg:152.71ms
step:751/1480 train_time:113099ms step_avg:152.63ms
step:752/1480 train_time:113255ms step_avg:152.63ms
step:753/1480 train_time:113412ms step_avg:152.64ms
step:754/1480 train_time:113568ms step_avg:152.65ms
step:755/1480 train_time:113724ms step_avg:152.65ms
step:756/1480 train_time:113879ms step_avg:152.65ms
step:757/1480 train_time:114038ms step_avg:152.66ms
step:758/1480 train_time:114195ms step_avg:152.67ms
step:759/1480 train_time:114367ms step_avg:152.69ms
step:760/1480 train_time:114509ms step_avg:152.68ms
step:761/1480 train_time:114666ms step_avg:152.68ms
step:762/1480 train_time:114822ms step_avg:152.69ms
step:763/1480 train_time:114979ms step_avg:152.69ms
step:764/1480 train_time:115136ms step_avg:152.70ms
step:765/1480 train_time:115293ms step_avg:152.71ms
step:766/1480 train_time:115452ms step_avg:152.71ms
step:767/1480 train_time:115609ms step_avg:152.72ms
step:768/1480 train_time:115765ms step_avg:152.72ms
step:769/1480 train_time:115921ms step_avg:152.73ms
step:770/1480 train_time:116079ms step_avg:152.74ms
step:771/1480 train_time:116238ms step_avg:152.74ms
step:772/1480 train_time:116395ms step_avg:152.75ms
step:773/1480 train_time:116552ms step_avg:152.76ms
step:774/1480 train_time:116710ms step_avg:152.76ms
step:775/1480 train_time:116869ms step_avg:152.77ms
step:776/1480 train_time:117028ms step_avg:152.78ms
step:777/1480 train_time:117188ms step_avg:152.79ms
step:778/1480 train_time:117347ms step_avg:152.80ms
step:779/1480 train_time:117504ms step_avg:152.80ms
step:780/1480 train_time:117660ms step_avg:152.81ms
step:781/1480 train_time:117819ms step_avg:152.81ms
step:782/1480 train_time:117978ms step_avg:152.82ms
step:783/1480 train_time:118135ms step_avg:152.83ms
step:784/1480 train_time:118294ms step_avg:152.83ms
step:785/1480 train_time:118452ms step_avg:152.84ms
step:786/1480 train_time:118609ms step_avg:152.85ms
step:787/1480 train_time:118766ms step_avg:152.85ms
step:788/1480 train_time:118923ms step_avg:152.86ms
step:789/1480 train_time:119080ms step_avg:152.86ms
step:790/1480 train_time:119237ms step_avg:152.87ms
step:791/1480 train_time:119399ms step_avg:152.88ms
step:792/1480 train_time:119556ms step_avg:152.89ms
step:793/1480 train_time:119713ms step_avg:152.89ms
step:794/1480 train_time:119871ms step_avg:152.90ms
step:795/1480 train_time:120030ms step_avg:152.91ms
step:796/1480 train_time:120192ms step_avg:152.92ms
step:797/1480 train_time:120353ms step_avg:152.93ms
step:798/1480 train_time:120513ms step_avg:152.94ms
step:799/1480 train_time:120675ms step_avg:152.95ms
step:800/1480 train_time:120832ms step_avg:152.95ms
step:801/1480 train_time:120989ms step_avg:152.96ms
step:802/1480 train_time:121147ms step_avg:152.96ms
step:803/1480 train_time:121305ms step_avg:152.97ms
step:804/1480 train_time:121462ms step_avg:152.97ms
step:805/1480 train_time:121621ms step_avg:152.98ms
step:806/1480 train_time:121777ms step_avg:152.99ms
step:807/1480 train_time:121934ms step_avg:152.99ms
step:808/1480 train_time:122093ms step_avg:153.00ms
step:809/1480 train_time:122251ms step_avg:153.00ms
step:810/1480 train_time:122408ms step_avg:153.01ms
step:811/1480 train_time:122565ms step_avg:153.02ms
step:812/1480 train_time:122722ms step_avg:153.02ms
step:813/1480 train_time:122879ms step_avg:153.02ms
step:814/1480 train_time:123037ms step_avg:153.03ms
step:815/1480 train_time:123194ms step_avg:153.04ms
step:816/1480 train_time:123354ms step_avg:153.04ms
step:817/1480 train_time:123512ms step_avg:153.05ms
step:818/1480 train_time:123670ms step_avg:153.06ms
step:819/1480 train_time:123827ms step_avg:153.06ms
step:820/1480 train_time:123987ms step_avg:153.07ms
step:821/1480 train_time:124144ms step_avg:153.08ms
step:822/1480 train_time:124301ms step_avg:153.08ms
step:823/1480 train_time:124459ms step_avg:153.09ms
step:824/1480 train_time:124617ms step_avg:153.09ms
step:825/1480 train_time:124776ms step_avg:153.10ms
step:826/1480 train_time:124936ms step_avg:153.11ms
step:827/1480 train_time:125094ms step_avg:153.11ms
step:828/1480 train_time:125253ms step_avg:153.12ms
step:829/1480 train_time:125412ms step_avg:153.13ms
step:830/1480 train_time:125572ms step_avg:153.14ms
step:831/1480 train_time:125730ms step_avg:153.14ms
step:832/1480 train_time:125888ms step_avg:153.15ms
step:833/1480 train_time:126046ms step_avg:153.15ms
step:834/1480 train_time:126205ms step_avg:153.16ms
step:835/1480 train_time:126363ms step_avg:153.17ms
step:836/1480 train_time:126521ms step_avg:153.17ms
step:837/1480 train_time:126679ms step_avg:153.18ms
step:838/1480 train_time:126837ms step_avg:153.18ms
step:839/1480 train_time:126995ms step_avg:153.19ms
step:840/1480 train_time:127152ms step_avg:153.20ms
step:841/1480 train_time:127311ms step_avg:153.20ms
step:842/1480 train_time:127470ms step_avg:153.21ms
step:843/1480 train_time:127627ms step_avg:153.21ms
step:844/1480 train_time:127784ms step_avg:153.22ms
step:845/1480 train_time:127940ms step_avg:153.22ms
step:846/1480 train_time:128099ms step_avg:153.23ms
step:847/1480 train_time:128257ms step_avg:153.23ms
step:848/1480 train_time:128417ms step_avg:153.24ms
step:849/1480 train_time:128576ms step_avg:153.25ms
step:850/1480 train_time:128735ms step_avg:153.26ms
step:851/1480 train_time:128894ms step_avg:153.26ms
step:852/1480 train_time:129052ms step_avg:153.27ms
step:853/1480 train_time:129209ms step_avg:153.27ms
step:854/1480 train_time:129366ms step_avg:153.28ms
step:855/1480 train_time:129523ms step_avg:153.28ms
step:856/1480 train_time:129680ms step_avg:153.29ms
step:857/1480 train_time:129838ms step_avg:153.29ms
step:858/1480 train_time:129998ms step_avg:153.30ms
step:859/1480 train_time:130156ms step_avg:153.30ms
step:860/1480 train_time:130313ms step_avg:153.31ms
step:861/1480 train_time:130473ms step_avg:153.32ms
step:862/1480 train_time:130636ms step_avg:153.33ms
step:863/1480 train_time:130794ms step_avg:153.33ms
step:864/1480 train_time:130952ms step_avg:153.34ms
step:865/1480 train_time:131111ms step_avg:153.35ms
step:866/1480 train_time:131269ms step_avg:153.35ms
step:867/1480 train_time:131427ms step_avg:153.36ms
step:868/1480 train_time:131583ms step_avg:153.36ms
step:869/1480 train_time:131740ms step_avg:153.36ms
step:870/1480 train_time:131899ms step_avg:153.37ms
step:871/1480 train_time:132055ms step_avg:153.37ms
step:872/1480 train_time:132213ms step_avg:153.38ms
step:873/1480 train_time:132370ms step_avg:153.38ms
step:874/1480 train_time:132531ms step_avg:153.39ms
step:875/1480 train_time:132691ms step_avg:153.40ms
step:875/1480 val_loss:3.5020 train_time:132764ms step_avg:153.48ms
step:876/1480 train_time:132855ms step_avg:153.41ms
step:877/1480 train_time:133011ms step_avg:153.42ms
step:878/1480 train_time:133170ms step_avg:153.42ms
step:879/1480 train_time:133329ms step_avg:153.43ms
step:880/1480 train_time:133487ms step_avg:153.43ms
step:881/1480 train_time:133645ms step_avg:153.44ms
step:882/1480 train_time:133805ms step_avg:153.45ms
step:883/1480 train_time:133965ms step_avg:153.45ms
step:884/1480 train_time:134126ms step_avg:153.46ms
step:885/1480 train_time:134286ms step_avg:153.47ms
step:886/1480 train_time:134447ms step_avg:153.48ms
step:887/1480 train_time:134607ms step_avg:153.49ms
step:888/1480 train_time:134770ms step_avg:153.50ms
step:889/1480 train_time:134931ms step_avg:153.51ms
step:890/1480 train_time:135089ms step_avg:153.51ms
step:891/1480 train_time:135248ms step_avg:153.52ms
step:892/1480 train_time:135409ms step_avg:153.52ms
step:893/1480 train_time:135567ms step_avg:153.53ms
step:894/1480 train_time:135727ms step_avg:153.54ms
step:895/1480 train_time:135890ms step_avg:153.55ms
step:896/1480 train_time:136050ms step_avg:153.56ms
step:897/1480 train_time:136209ms step_avg:153.56ms
step:898/1480 train_time:136368ms step_avg:153.57ms
step:899/1480 train_time:136526ms step_avg:153.57ms
step:900/1480 train_time:136685ms step_avg:153.58ms
step:901/1480 train_time:136846ms step_avg:153.59ms
step:902/1480 train_time:137003ms step_avg:153.59ms
step:903/1480 train_time:137166ms step_avg:153.60ms
step:904/1480 train_time:137325ms step_avg:153.61ms
step:905/1480 train_time:137483ms step_avg:153.61ms
step:906/1480 train_time:137643ms step_avg:153.62ms
step:907/1480 train_time:137808ms step_avg:153.63ms
step:908/1480 train_time:137966ms step_avg:153.64ms
step:909/1480 train_time:138126ms step_avg:153.64ms
step:910/1480 train_time:138291ms step_avg:153.66ms
step:911/1480 train_time:138449ms step_avg:153.66ms
step:912/1480 train_time:138608ms step_avg:153.67ms
step:913/1480 train_time:138770ms step_avg:153.68ms
step:914/1480 train_time:138929ms step_avg:153.68ms
step:915/1480 train_time:139093ms step_avg:153.69ms
step:916/1480 train_time:139252ms step_avg:153.70ms
step:917/1480 train_time:139411ms step_avg:153.71ms
step:918/1480 train_time:139572ms step_avg:153.71ms
step:919/1480 train_time:139732ms step_avg:153.72ms
step:920/1480 train_time:139891ms step_avg:153.73ms
step:921/1480 train_time:140050ms step_avg:153.73ms
step:922/1480 train_time:140212ms step_avg:153.74ms
step:923/1480 train_time:140370ms step_avg:153.75ms
step:924/1480 train_time:140529ms step_avg:153.75ms
step:925/1480 train_time:140689ms step_avg:153.76ms
step:926/1480 train_time:140847ms step_avg:153.76ms
step:927/1480 train_time:141005ms step_avg:153.77ms
step:928/1480 train_time:141166ms step_avg:153.78ms
step:929/1480 train_time:141327ms step_avg:153.78ms
step:930/1480 train_time:141487ms step_avg:153.79ms
step:931/1480 train_time:141646ms step_avg:153.80ms
step:932/1480 train_time:141804ms step_avg:153.80ms
step:933/1480 train_time:141964ms step_avg:153.81ms
step:934/1480 train_time:142123ms step_avg:153.81ms
step:935/1480 train_time:142283ms step_avg:153.82ms
step:936/1480 train_time:142442ms step_avg:153.83ms
step:937/1480 train_time:142604ms step_avg:153.83ms
step:938/1480 train_time:142763ms step_avg:153.84ms
step:939/1480 train_time:142924ms step_avg:153.85ms
step:940/1480 train_time:143086ms step_avg:153.86ms
step:941/1480 train_time:143245ms step_avg:153.86ms
step:942/1480 train_time:143403ms step_avg:153.87ms
step:943/1480 train_time:143565ms step_avg:153.87ms
step:944/1480 train_time:143727ms step_avg:153.88ms
step:945/1480 train_time:143886ms step_avg:153.89ms
step:946/1480 train_time:144049ms step_avg:153.90ms
step:947/1480 train_time:144210ms step_avg:153.91ms
step:948/1480 train_time:144368ms step_avg:153.91ms
step:949/1480 train_time:144551ms step_avg:153.94ms
step:950/1480 train_time:144687ms step_avg:153.92ms
step:951/1480 train_time:144850ms step_avg:153.93ms
step:952/1480 train_time:145010ms step_avg:153.94ms
step:953/1480 train_time:145169ms step_avg:153.94ms
step:954/1480 train_time:145331ms step_avg:153.95ms
step:955/1480 train_time:145491ms step_avg:153.96ms
step:956/1480 train_time:145649ms step_avg:153.96ms
step:957/1480 train_time:145810ms step_avg:153.97ms
step:958/1480 train_time:145974ms step_avg:153.98ms
step:959/1480 train_time:146131ms step_avg:153.98ms
step:960/1480 train_time:146291ms step_avg:153.99ms
step:961/1480 train_time:146450ms step_avg:154.00ms
step:962/1480 train_time:146608ms step_avg:154.00ms
step:963/1480 train_time:146769ms step_avg:154.01ms
step:964/1480 train_time:146929ms step_avg:154.01ms
step:965/1480 train_time:147088ms step_avg:154.02ms
step:966/1480 train_time:147247ms step_avg:154.02ms
step:967/1480 train_time:147405ms step_avg:154.03ms
step:968/1480 train_time:147565ms step_avg:154.03ms
step:969/1480 train_time:147725ms step_avg:154.04ms
step:970/1480 train_time:147884ms step_avg:154.05ms
step:971/1480 train_time:148045ms step_avg:154.05ms
step:972/1480 train_time:148203ms step_avg:154.06ms
step:973/1480 train_time:148362ms step_avg:154.06ms
step:974/1480 train_time:148521ms step_avg:154.07ms
step:975/1480 train_time:148682ms step_avg:154.07ms
step:976/1480 train_time:148843ms step_avg:154.08ms
step:977/1480 train_time:149002ms step_avg:154.09ms
step:978/1480 train_time:149162ms step_avg:154.09ms
step:979/1480 train_time:149323ms step_avg:154.10ms
step:980/1480 train_time:149483ms step_avg:154.11ms
step:981/1480 train_time:149643ms step_avg:154.11ms
step:982/1480 train_time:149799ms step_avg:154.11ms
step:983/1480 train_time:149959ms step_avg:154.12ms
step:984/1480 train_time:150119ms step_avg:154.13ms
step:985/1480 train_time:150279ms step_avg:154.13ms
step:986/1480 train_time:150440ms step_avg:154.14ms
step:987/1480 train_time:150598ms step_avg:154.14ms
step:988/1480 train_time:150758ms step_avg:154.15ms
step:989/1480 train_time:150917ms step_avg:154.15ms
step:990/1480 train_time:151081ms step_avg:154.16ms
step:991/1480 train_time:151243ms step_avg:154.17ms
step:992/1480 train_time:151408ms step_avg:154.18ms
step:993/1480 train_time:151575ms step_avg:154.20ms
step:994/1480 train_time:151733ms step_avg:154.20ms
step:995/1480 train_time:151892ms step_avg:154.21ms
step:996/1480 train_time:152050ms step_avg:154.21ms
step:997/1480 train_time:152211ms step_avg:154.22ms
step:998/1480 train_time:152369ms step_avg:154.22ms
step:999/1480 train_time:152530ms step_avg:154.23ms
step:1000/1480 train_time:152693ms step_avg:154.24ms
step:1000/1480 val_loss:3.4381 train_time:152766ms step_avg:154.31ms
step:1001/1480 train_time:152856ms step_avg:154.24ms
step:1002/1480 train_time:153017ms step_avg:154.25ms
step:1003/1480 train_time:153180ms step_avg:154.26ms
step:1004/1480 train_time:153343ms step_avg:154.27ms
step:1005/1480 train_time:153503ms step_avg:154.27ms
step:1006/1480 train_time:153663ms step_avg:154.28ms
step:1007/1480 train_time:153824ms step_avg:154.29ms
step:1008/1480 train_time:153985ms step_avg:154.29ms
step:1009/1480 train_time:154149ms step_avg:154.30ms
step:1010/1480 train_time:154308ms step_avg:154.31ms
step:1011/1480 train_time:154468ms step_avg:154.31ms
step:1012/1480 train_time:154626ms step_avg:154.32ms
step:1013/1480 train_time:154789ms step_avg:154.33ms
step:1014/1480 train_time:154948ms step_avg:154.33ms
step:1015/1480 train_time:155110ms step_avg:154.34ms
step:1016/1480 train_time:155270ms step_avg:154.34ms
step:1017/1480 train_time:155433ms step_avg:154.35ms
step:1018/1480 train_time:155593ms step_avg:154.36ms
step:1019/1480 train_time:155753ms step_avg:154.36ms
step:1020/1480 train_time:155916ms step_avg:154.37ms
step:1021/1480 train_time:156077ms step_avg:154.38ms
step:1022/1480 train_time:156237ms step_avg:154.38ms
step:1023/1480 train_time:156398ms step_avg:154.39ms
step:1024/1480 train_time:156557ms step_avg:154.40ms
step:1025/1480 train_time:156719ms step_avg:154.40ms
step:1026/1480 train_time:156880ms step_avg:154.41ms
step:1027/1480 train_time:157039ms step_avg:154.41ms
step:1028/1480 train_time:157202ms step_avg:154.42ms
step:1029/1480 train_time:157365ms step_avg:154.43ms
step:1030/1480 train_time:157526ms step_avg:154.44ms
step:1031/1480 train_time:157683ms step_avg:154.44ms
step:1032/1480 train_time:157848ms step_avg:154.45ms
step:1033/1480 train_time:158007ms step_avg:154.45ms
step:1034/1480 train_time:158169ms step_avg:154.46ms
step:1035/1480 train_time:158328ms step_avg:154.47ms
step:1036/1480 train_time:158487ms step_avg:154.47ms
step:1037/1480 train_time:158647ms step_avg:154.48ms
step:1038/1480 train_time:158806ms step_avg:154.48ms
step:1039/1480 train_time:158968ms step_avg:154.49ms
step:1040/1480 train_time:159129ms step_avg:154.49ms
step:1041/1480 train_time:159288ms step_avg:154.50ms
step:1042/1480 train_time:159446ms step_avg:154.50ms
step:1043/1480 train_time:159605ms step_avg:154.51ms
step:1044/1480 train_time:159765ms step_avg:154.51ms
step:1045/1480 train_time:159925ms step_avg:154.52ms
step:1046/1480 train_time:160086ms step_avg:154.52ms
step:1047/1480 train_time:160245ms step_avg:154.53ms
step:1048/1480 train_time:160405ms step_avg:154.53ms
step:1049/1480 train_time:160565ms step_avg:154.54ms
step:1050/1480 train_time:160726ms step_avg:154.54ms
step:1051/1480 train_time:160888ms step_avg:154.55ms
step:1052/1480 train_time:161049ms step_avg:154.56ms
step:1053/1480 train_time:161209ms step_avg:154.56ms
step:1054/1480 train_time:161369ms step_avg:154.57ms
step:1055/1480 train_time:161529ms step_avg:154.57ms
step:1056/1480 train_time:161687ms step_avg:154.58ms
step:1057/1480 train_time:161848ms step_avg:154.58ms
step:1058/1480 train_time:162011ms step_avg:154.59ms
step:1059/1480 train_time:162174ms step_avg:154.60ms
step:1060/1480 train_time:162337ms step_avg:154.61ms
step:1061/1480 train_time:162498ms step_avg:154.61ms
step:1062/1480 train_time:162659ms step_avg:154.62ms
step:1063/1480 train_time:162820ms step_avg:154.62ms
step:1064/1480 train_time:162978ms step_avg:154.63ms
step:1065/1480 train_time:163139ms step_avg:154.63ms
step:1066/1480 train_time:163304ms step_avg:154.64ms
step:1067/1480 train_time:163464ms step_avg:154.65ms
step:1068/1480 train_time:163624ms step_avg:154.65ms
step:1069/1480 train_time:163786ms step_avg:154.66ms
step:1070/1480 train_time:163946ms step_avg:154.67ms
step:1071/1480 train_time:164110ms step_avg:154.67ms
step:1072/1480 train_time:164269ms step_avg:154.68ms
step:1073/1480 train_time:164427ms step_avg:154.68ms
step:1074/1480 train_time:164586ms step_avg:154.69ms
step:1075/1480 train_time:164747ms step_avg:154.69ms
step:1076/1480 train_time:164907ms step_avg:154.70ms
step:1077/1480 train_time:165066ms step_avg:154.70ms
step:1078/1480 train_time:165230ms step_avg:154.71ms
step:1079/1480 train_time:165394ms step_avg:154.72ms
step:1080/1480 train_time:165554ms step_avg:154.72ms
step:1081/1480 train_time:165716ms step_avg:154.73ms
step:1082/1480 train_time:165877ms step_avg:154.74ms
step:1083/1480 train_time:166038ms step_avg:154.74ms
step:1084/1480 train_time:166199ms step_avg:154.75ms
step:1085/1480 train_time:166358ms step_avg:154.75ms
step:1086/1480 train_time:166519ms step_avg:154.76ms
step:1087/1480 train_time:166680ms step_avg:154.76ms
step:1088/1480 train_time:166843ms step_avg:154.77ms
step:1089/1480 train_time:167007ms step_avg:154.78ms
step:1090/1480 train_time:167171ms step_avg:154.79ms
step:1091/1480 train_time:167331ms step_avg:154.79ms
step:1092/1480 train_time:167491ms step_avg:154.80ms
step:1093/1480 train_time:167650ms step_avg:154.80ms
step:1094/1480 train_time:167810ms step_avg:154.81ms
step:1095/1480 train_time:167969ms step_avg:154.81ms
step:1096/1480 train_time:168133ms step_avg:154.82ms
step:1097/1480 train_time:168296ms step_avg:154.83ms
step:1098/1480 train_time:168459ms step_avg:154.83ms
step:1099/1480 train_time:168621ms step_avg:154.84ms
step:1100/1480 train_time:168783ms step_avg:154.85ms
step:1101/1480 train_time:168947ms step_avg:154.86ms
step:1102/1480 train_time:169109ms step_avg:154.86ms
step:1103/1480 train_time:169277ms step_avg:154.87ms
step:1104/1480 train_time:169440ms step_avg:154.88ms
step:1105/1480 train_time:169602ms step_avg:154.89ms
step:1106/1480 train_time:169763ms step_avg:154.89ms
step:1107/1480 train_time:169923ms step_avg:154.90ms
step:1108/1480 train_time:170083ms step_avg:154.90ms
step:1109/1480 train_time:170244ms step_avg:154.91ms
step:1110/1480 train_time:170405ms step_avg:154.91ms
step:1111/1480 train_time:170566ms step_avg:154.92ms
step:1112/1480 train_time:170728ms step_avg:154.93ms
step:1113/1480 train_time:170897ms step_avg:154.94ms
step:1114/1480 train_time:171060ms step_avg:154.95ms
step:1115/1480 train_time:171223ms step_avg:154.95ms
step:1116/1480 train_time:171382ms step_avg:154.96ms
step:1117/1480 train_time:171545ms step_avg:154.96ms
step:1118/1480 train_time:171710ms step_avg:154.97ms
step:1119/1480 train_time:171870ms step_avg:154.98ms
step:1120/1480 train_time:172031ms step_avg:154.98ms
step:1121/1480 train_time:172193ms step_avg:154.99ms
step:1122/1480 train_time:172352ms step_avg:154.99ms
step:1123/1480 train_time:172513ms step_avg:155.00ms
step:1124/1480 train_time:172677ms step_avg:155.01ms
step:1125/1480 train_time:172839ms step_avg:155.01ms
step:1125/1480 val_loss:3.3823 train_time:172915ms step_avg:155.08ms
step:1126/1480 train_time:173006ms step_avg:155.02ms
step:1127/1480 train_time:173166ms step_avg:155.03ms
step:1128/1480 train_time:173326ms step_avg:155.03ms
step:1129/1480 train_time:173491ms step_avg:155.04ms
step:1130/1480 train_time:173652ms step_avg:155.05ms
step:1131/1480 train_time:173818ms step_avg:155.06ms
step:1132/1480 train_time:173976ms step_avg:155.06ms
step:1133/1480 train_time:174140ms step_avg:155.07ms
step:1134/1480 train_time:174303ms step_avg:155.07ms
step:1135/1480 train_time:174465ms step_avg:155.08ms
step:1136/1480 train_time:174629ms step_avg:155.09ms
step:1137/1480 train_time:174789ms step_avg:155.09ms
step:1138/1480 train_time:174954ms step_avg:155.10ms
step:1139/1480 train_time:175131ms step_avg:155.12ms
step:1140/1480 train_time:175277ms step_avg:155.11ms
step:1141/1480 train_time:175441ms step_avg:155.12ms
step:1142/1480 train_time:175601ms step_avg:155.12ms
step:1143/1480 train_time:175766ms step_avg:155.13ms
step:1144/1480 train_time:175929ms step_avg:155.14ms
step:1145/1480 train_time:176089ms step_avg:155.14ms
step:1146/1480 train_time:176254ms step_avg:155.15ms
step:1147/1480 train_time:176413ms step_avg:155.16ms
step:1148/1480 train_time:176575ms step_avg:155.16ms
step:1149/1480 train_time:176737ms step_avg:155.17ms
step:1150/1480 train_time:176896ms step_avg:155.17ms
step:1151/1480 train_time:177061ms step_avg:155.18ms
step:1152/1480 train_time:177225ms step_avg:155.19ms
step:1153/1480 train_time:177391ms step_avg:155.20ms
step:1154/1480 train_time:177553ms step_avg:155.20ms
step:1155/1480 train_time:177714ms step_avg:155.21ms
step:1156/1480 train_time:177880ms step_avg:155.22ms
step:1157/1480 train_time:178043ms step_avg:155.23ms
step:1158/1480 train_time:178204ms step_avg:155.23ms
step:1159/1480 train_time:178365ms step_avg:155.23ms
step:1160/1480 train_time:178529ms step_avg:155.24ms
step:1161/1480 train_time:178692ms step_avg:155.25ms
step:1162/1480 train_time:178855ms step_avg:155.26ms
step:1163/1480 train_time:179016ms step_avg:155.26ms
step:1164/1480 train_time:179177ms step_avg:155.27ms
step:1165/1480 train_time:179335ms step_avg:155.27ms
step:1166/1480 train_time:179499ms step_avg:155.28ms
step:1167/1480 train_time:179660ms step_avg:155.28ms
step:1168/1480 train_time:179823ms step_avg:155.29ms
step:1169/1480 train_time:179986ms step_avg:155.29ms
step:1170/1480 train_time:180147ms step_avg:155.30ms
step:1171/1480 train_time:180309ms step_avg:155.30ms
step:1172/1480 train_time:180469ms step_avg:155.31ms
step:1173/1480 train_time:180631ms step_avg:155.31ms
step:1174/1480 train_time:180800ms step_avg:155.33ms
step:1175/1480 train_time:180963ms step_avg:155.33ms
step:1176/1480 train_time:181127ms step_avg:155.34ms
step:1177/1480 train_time:181294ms step_avg:155.35ms
step:1178/1480 train_time:181454ms step_avg:155.35ms
step:1179/1480 train_time:181614ms step_avg:155.36ms
step:1180/1480 train_time:181783ms step_avg:155.37ms
step:1181/1480 train_time:181948ms step_avg:155.38ms
step:1182/1480 train_time:182108ms step_avg:155.38ms
step:1183/1480 train_time:182270ms step_avg:155.39ms
step:1184/1480 train_time:182430ms step_avg:155.39ms
step:1185/1480 train_time:182595ms step_avg:155.40ms
step:1186/1480 train_time:182757ms step_avg:155.41ms
step:1187/1480 train_time:182929ms step_avg:155.42ms
step:1188/1480 train_time:183090ms step_avg:155.42ms
step:1189/1480 train_time:183252ms step_avg:155.43ms
step:1190/1480 train_time:183414ms step_avg:155.44ms
step:1191/1480 train_time:183578ms step_avg:155.44ms
step:1192/1480 train_time:183738ms step_avg:155.45ms
step:1193/1480 train_time:183898ms step_avg:155.45ms
step:1194/1480 train_time:184059ms step_avg:155.45ms
step:1195/1480 train_time:184222ms step_avg:155.46ms
step:1196/1480 train_time:184394ms step_avg:155.48ms
step:1197/1480 train_time:184556ms step_avg:155.48ms
step:1198/1480 train_time:184725ms step_avg:155.49ms
step:1199/1480 train_time:184889ms step_avg:155.50ms
step:1200/1480 train_time:185050ms step_avg:155.50ms
step:1201/1480 train_time:185210ms step_avg:155.51ms
step:1202/1480 train_time:185379ms step_avg:155.52ms
step:1203/1480 train_time:185545ms step_avg:155.53ms
step:1204/1480 train_time:185709ms step_avg:155.53ms
step:1205/1480 train_time:185871ms step_avg:155.54ms
step:1206/1480 train_time:186031ms step_avg:155.54ms
step:1207/1480 train_time:186193ms step_avg:155.55ms
step:1208/1480 train_time:186353ms step_avg:155.55ms
step:1209/1480 train_time:186516ms step_avg:155.56ms
step:1210/1480 train_time:186681ms step_avg:155.57ms
step:1211/1480 train_time:186845ms step_avg:155.57ms
step:1212/1480 train_time:187011ms step_avg:155.58ms
step:1213/1480 train_time:187176ms step_avg:155.59ms
step:1214/1480 train_time:187341ms step_avg:155.60ms
step:1215/1480 train_time:187505ms step_avg:155.61ms
step:1216/1480 train_time:187666ms step_avg:155.61ms
step:1217/1480 train_time:187829ms step_avg:155.62ms
step:1218/1480 train_time:187993ms step_avg:155.62ms
step:1219/1480 train_time:188158ms step_avg:155.63ms
step:1220/1480 train_time:188320ms step_avg:155.64ms
step:1221/1480 train_time:188482ms step_avg:155.64ms
step:1222/1480 train_time:188642ms step_avg:155.65ms
step:1223/1480 train_time:188805ms step_avg:155.65ms
step:1224/1480 train_time:188973ms step_avg:155.66ms
step:1225/1480 train_time:189136ms step_avg:155.67ms
step:1226/1480 train_time:189300ms step_avg:155.67ms
step:1227/1480 train_time:189465ms step_avg:155.68ms
step:1228/1480 train_time:189629ms step_avg:155.69ms
step:1229/1480 train_time:189792ms step_avg:155.69ms
step:1230/1480 train_time:189960ms step_avg:155.70ms
step:1231/1480 train_time:190126ms step_avg:155.71ms
step:1232/1480 train_time:190291ms step_avg:155.72ms
step:1233/1480 train_time:190452ms step_avg:155.72ms
step:1234/1480 train_time:190614ms step_avg:155.73ms
step:1235/1480 train_time:190779ms step_avg:155.74ms
step:1236/1480 train_time:190939ms step_avg:155.74ms
step:1237/1480 train_time:191100ms step_avg:155.75ms
step:1238/1480 train_time:191275ms step_avg:155.76ms
step:1239/1480 train_time:191436ms step_avg:155.77ms
step:1240/1480 train_time:191598ms step_avg:155.77ms
step:1241/1480 train_time:191763ms step_avg:155.78ms
step:1242/1480 train_time:191925ms step_avg:155.78ms
step:1243/1480 train_time:192089ms step_avg:155.79ms
step:1244/1480 train_time:192251ms step_avg:155.79ms
step:1245/1480 train_time:192413ms step_avg:155.80ms
step:1246/1480 train_time:192575ms step_avg:155.81ms
step:1247/1480 train_time:192737ms step_avg:155.81ms
step:1248/1480 train_time:192898ms step_avg:155.81ms
step:1249/1480 train_time:193060ms step_avg:155.82ms
step:1250/1480 train_time:193220ms step_avg:155.82ms
step:1250/1480 val_loss:3.3332 train_time:193296ms step_avg:155.88ms
step:1251/1480 train_time:193389ms step_avg:155.83ms
step:1252/1480 train_time:193551ms step_avg:155.84ms
step:1253/1480 train_time:193713ms step_avg:155.84ms
step:1254/1480 train_time:193873ms step_avg:155.85ms
step:1255/1480 train_time:194045ms step_avg:155.86ms
step:1256/1480 train_time:194210ms step_avg:155.87ms
step:1257/1480 train_time:194370ms step_avg:155.87ms
step:1258/1480 train_time:194535ms step_avg:155.88ms
step:1259/1480 train_time:194698ms step_avg:155.88ms
step:1260/1480 train_time:194858ms step_avg:155.89ms
step:1261/1480 train_time:195023ms step_avg:155.89ms
step:1262/1480 train_time:195187ms step_avg:155.90ms
step:1263/1480 train_time:195351ms step_avg:155.91ms
step:1264/1480 train_time:195511ms step_avg:155.91ms
step:1265/1480 train_time:195671ms step_avg:155.91ms
step:1266/1480 train_time:195834ms step_avg:155.92ms
step:1267/1480 train_time:195994ms step_avg:155.92ms
step:1268/1480 train_time:196159ms step_avg:155.93ms
step:1269/1480 train_time:196325ms step_avg:155.94ms
step:1270/1480 train_time:196488ms step_avg:155.94ms
step:1271/1480 train_time:196651ms step_avg:155.95ms
step:1272/1480 train_time:196811ms step_avg:155.95ms
step:1273/1480 train_time:196973ms step_avg:155.96ms
step:1274/1480 train_time:197136ms step_avg:155.96ms
step:1275/1480 train_time:197297ms step_avg:155.97ms
step:1276/1480 train_time:197457ms step_avg:155.97ms
step:1277/1480 train_time:197620ms step_avg:155.97ms
step:1278/1480 train_time:197782ms step_avg:155.98ms
step:1279/1480 train_time:197944ms step_avg:155.98ms
step:1280/1480 train_time:198111ms step_avg:155.99ms
step:1281/1480 train_time:198273ms step_avg:156.00ms
step:1282/1480 train_time:198432ms step_avg:156.00ms
step:1283/1480 train_time:198595ms step_avg:156.01ms
step:1284/1480 train_time:198758ms step_avg:156.01ms
step:1285/1480 train_time:198919ms step_avg:156.01ms
step:1286/1480 train_time:199081ms step_avg:156.02ms
step:1287/1480 train_time:199242ms step_avg:156.02ms
step:1288/1480 train_time:199405ms step_avg:156.03ms
step:1289/1480 train_time:199574ms step_avg:156.04ms
step:1290/1480 train_time:199744ms step_avg:156.05ms
step:1291/1480 train_time:199908ms step_avg:156.06ms
step:1292/1480 train_time:200070ms step_avg:156.06ms
step:1293/1480 train_time:200238ms step_avg:156.07ms
step:1294/1480 train_time:200402ms step_avg:156.08ms
step:1295/1480 train_time:200566ms step_avg:156.08ms
step:1296/1480 train_time:200730ms step_avg:156.09ms
step:1297/1480 train_time:200892ms step_avg:156.09ms
step:1298/1480 train_time:201055ms step_avg:156.10ms
step:1299/1480 train_time:201218ms step_avg:156.10ms
step:1300/1480 train_time:201380ms step_avg:156.11ms
step:1301/1480 train_time:201542ms step_avg:156.11ms
step:1302/1480 train_time:201708ms step_avg:156.12ms
step:1303/1480 train_time:201873ms step_avg:156.13ms
step:1304/1480 train_time:202038ms step_avg:156.13ms
step:1305/1480 train_time:202200ms step_avg:156.14ms
step:1306/1480 train_time:202365ms step_avg:156.15ms
step:1307/1480 train_time:202526ms step_avg:156.15ms
step:1308/1480 train_time:202688ms step_avg:156.15ms
step:1309/1480 train_time:202851ms step_avg:156.16ms
step:1310/1480 train_time:203014ms step_avg:156.16ms
step:1311/1480 train_time:203174ms step_avg:156.17ms
step:1312/1480 train_time:203342ms step_avg:156.18ms
step:1313/1480 train_time:203505ms step_avg:156.18ms
step:1314/1480 train_time:203668ms step_avg:156.19ms
step:1315/1480 train_time:203831ms step_avg:156.19ms
step:1316/1480 train_time:203990ms step_avg:156.19ms
step:1317/1480 train_time:204152ms step_avg:156.20ms
step:1318/1480 train_time:204319ms step_avg:156.21ms
step:1319/1480 train_time:204486ms step_avg:156.22ms
step:1320/1480 train_time:204654ms step_avg:156.22ms
step:1321/1480 train_time:204818ms step_avg:156.23ms
step:1322/1480 train_time:204989ms step_avg:156.24ms
step:1323/1480 train_time:205151ms step_avg:156.25ms
step:1324/1480 train_time:205317ms step_avg:156.25ms
step:1325/1480 train_time:205486ms step_avg:156.26ms
step:1326/1480 train_time:205652ms step_avg:156.27ms
step:1327/1480 train_time:205814ms step_avg:156.27ms
step:1328/1480 train_time:205975ms step_avg:156.28ms
step:1329/1480 train_time:206167ms step_avg:156.31ms
step:1330/1480 train_time:206324ms step_avg:156.31ms
step:1331/1480 train_time:206488ms step_avg:156.31ms
step:1332/1480 train_time:206650ms step_avg:156.32ms
step:1333/1480 train_time:206817ms step_avg:156.32ms
step:1334/1480 train_time:206980ms step_avg:156.33ms
step:1335/1480 train_time:207142ms step_avg:156.33ms
step:1336/1480 train_time:207311ms step_avg:156.34ms
step:1337/1480 train_time:207476ms step_avg:156.35ms
step:1338/1480 train_time:207639ms step_avg:156.35ms
step:1339/1480 train_time:207804ms step_avg:156.36ms
step:1340/1480 train_time:207966ms step_avg:156.37ms
step:1341/1480 train_time:208130ms step_avg:156.37ms
step:1342/1480 train_time:208297ms step_avg:156.38ms
step:1343/1480 train_time:208460ms step_avg:156.38ms
step:1344/1480 train_time:208622ms step_avg:156.39ms
step:1345/1480 train_time:208791ms step_avg:156.40ms
step:1346/1480 train_time:208952ms step_avg:156.40ms
step:1347/1480 train_time:209115ms step_avg:156.41ms
step:1348/1480 train_time:209277ms step_avg:156.41ms
step:1349/1480 train_time:209439ms step_avg:156.41ms
step:1350/1480 train_time:209605ms step_avg:156.42ms
step:1351/1480 train_time:209767ms step_avg:156.43ms
step:1352/1480 train_time:209930ms step_avg:156.43ms
step:1353/1480 train_time:210096ms step_avg:156.44ms
step:1354/1480 train_time:210259ms step_avg:156.44ms
step:1355/1480 train_time:210422ms step_avg:156.45ms
step:1356/1480 train_time:210587ms step_avg:156.45ms
step:1357/1480 train_time:210750ms step_avg:156.46ms
step:1358/1480 train_time:210915ms step_avg:156.47ms
step:1359/1480 train_time:211079ms step_avg:156.47ms
step:1360/1480 train_time:211245ms step_avg:156.48ms
step:1361/1480 train_time:211412ms step_avg:156.49ms
step:1362/1480 train_time:211577ms step_avg:156.49ms
step:1363/1480 train_time:211746ms step_avg:156.50ms
step:1364/1480 train_time:211908ms step_avg:156.51ms
step:1365/1480 train_time:212068ms step_avg:156.51ms
step:1366/1480 train_time:212234ms step_avg:156.51ms
step:1367/1480 train_time:212396ms step_avg:156.52ms
step:1368/1480 train_time:212563ms step_avg:156.53ms
step:1369/1480 train_time:212733ms step_avg:156.54ms
step:1370/1480 train_time:212900ms step_avg:156.54ms
step:1371/1480 train_time:213060ms step_avg:156.55ms
step:1372/1480 train_time:213228ms step_avg:156.55ms
step:1373/1480 train_time:213389ms step_avg:156.56ms
step:1374/1480 train_time:213556ms step_avg:156.57ms
step:1375/1480 train_time:213719ms step_avg:156.57ms
step:1375/1480 val_loss:3.2948 train_time:213794ms step_avg:156.63ms
step:1376/1480 train_time:213886ms step_avg:156.58ms
step:1377/1480 train_time:214048ms step_avg:156.58ms
step:1378/1480 train_time:214209ms step_avg:156.59ms
step:1379/1480 train_time:214374ms step_avg:156.59ms
step:1380/1480 train_time:214538ms step_avg:156.60ms
step:1381/1480 train_time:214705ms step_avg:156.60ms
step:1382/1480 train_time:214870ms step_avg:156.61ms
step:1383/1480 train_time:215032ms step_avg:156.61ms
step:1384/1480 train_time:215199ms step_avg:156.62ms
step:1385/1480 train_time:215360ms step_avg:156.63ms
step:1386/1480 train_time:215525ms step_avg:156.63ms
step:1387/1480 train_time:215691ms step_avg:156.64ms
step:1388/1480 train_time:215851ms step_avg:156.64ms
step:1389/1480 train_time:216017ms step_avg:156.65ms
step:1390/1480 train_time:216178ms step_avg:156.65ms
step:1391/1480 train_time:216341ms step_avg:156.66ms
step:1392/1480 train_time:216505ms step_avg:156.66ms
step:1393/1480 train_time:216668ms step_avg:156.67ms
step:1394/1480 train_time:216830ms step_avg:156.67ms
step:1395/1480 train_time:216991ms step_avg:156.67ms
step:1396/1480 train_time:217154ms step_avg:156.68ms
step:1397/1480 train_time:217315ms step_avg:156.68ms
step:1398/1480 train_time:217475ms step_avg:156.68ms
step:1399/1480 train_time:217637ms step_avg:156.69ms
step:1400/1480 train_time:217805ms step_avg:156.69ms
step:1401/1480 train_time:217966ms step_avg:156.70ms
step:1402/1480 train_time:218128ms step_avg:156.70ms
step:1403/1480 train_time:218294ms step_avg:156.71ms
step:1404/1480 train_time:218457ms step_avg:156.71ms
step:1405/1480 train_time:218623ms step_avg:156.72ms
step:1406/1480 train_time:218787ms step_avg:156.72ms
step:1407/1480 train_time:218948ms step_avg:156.73ms
step:1408/1480 train_time:219109ms step_avg:156.73ms
step:1409/1480 train_time:219280ms step_avg:156.74ms
step:1410/1480 train_time:219445ms step_avg:156.75ms
step:1411/1480 train_time:219606ms step_avg:156.75ms
step:1412/1480 train_time:219768ms step_avg:156.75ms
step:1413/1480 train_time:219930ms step_avg:156.76ms
step:1414/1480 train_time:220095ms step_avg:156.76ms
step:1415/1480 train_time:220260ms step_avg:156.77ms
step:1416/1480 train_time:220434ms step_avg:156.78ms
step:1417/1480 train_time:220599ms step_avg:156.79ms
step:1418/1480 train_time:220763ms step_avg:156.79ms
step:1419/1480 train_time:220928ms step_avg:156.80ms
step:1420/1480 train_time:221092ms step_avg:156.80ms
step:1421/1480 train_time:221256ms step_avg:156.81ms
step:1422/1480 train_time:221422ms step_avg:156.81ms
step:1423/1480 train_time:221584ms step_avg:156.82ms
step:1424/1480 train_time:221750ms step_avg:156.82ms
step:1425/1480 train_time:221921ms step_avg:156.83ms
step:1426/1480 train_time:222086ms step_avg:156.84ms
step:1427/1480 train_time:222252ms step_avg:156.85ms
step:1428/1480 train_time:222414ms step_avg:156.85ms
step:1429/1480 train_time:222574ms step_avg:156.85ms
step:1430/1480 train_time:222739ms step_avg:156.86ms
step:1431/1480 train_time:222905ms step_avg:156.86ms
step:1432/1480 train_time:223073ms step_avg:156.87ms
step:1433/1480 train_time:223243ms step_avg:156.88ms
step:1434/1480 train_time:223411ms step_avg:156.89ms
step:1435/1480 train_time:223576ms step_avg:156.90ms
step:1436/1480 train_time:223744ms step_avg:156.90ms
step:1437/1480 train_time:223906ms step_avg:156.91ms
step:1438/1480 train_time:224070ms step_avg:156.91ms
step:1439/1480 train_time:224237ms step_avg:156.92ms
step:1440/1480 train_time:224400ms step_avg:156.92ms
step:1441/1480 train_time:224566ms step_avg:156.93ms
step:1442/1480 train_time:224732ms step_avg:156.94ms
step:1443/1480 train_time:224905ms step_avg:156.95ms
step:1444/1480 train_time:225069ms step_avg:156.95ms
step:1445/1480 train_time:225231ms step_avg:156.96ms
step:1446/1480 train_time:225397ms step_avg:156.96ms
step:1447/1480 train_time:225565ms step_avg:156.97ms
step:1448/1480 train_time:225727ms step_avg:156.97ms
step:1449/1480 train_time:225890ms step_avg:156.98ms
step:1450/1480 train_time:226055ms step_avg:156.98ms
step:1451/1480 train_time:226218ms step_avg:156.99ms
step:1452/1480 train_time:226383ms step_avg:156.99ms
step:1453/1480 train_time:226546ms step_avg:157.00ms
step:1454/1480 train_time:226708ms step_avg:157.00ms
step:1455/1480 train_time:226876ms step_avg:157.01ms
step:1456/1480 train_time:227042ms step_avg:157.01ms
step:1457/1480 train_time:227205ms step_avg:157.02ms
step:1458/1480 train_time:227369ms step_avg:157.02ms
step:1459/1480 train_time:227534ms step_avg:157.03ms
step:1460/1480 train_time:227697ms step_avg:157.03ms
step:1461/1480 train_time:227863ms step_avg:157.04ms
step:1462/1480 train_time:228026ms step_avg:157.04ms
step:1463/1480 train_time:228191ms step_avg:157.05ms
step:1464/1480 train_time:228358ms step_avg:157.05ms
step:1465/1480 train_time:228523ms step_avg:157.06ms
step:1466/1480 train_time:228687ms step_avg:157.06ms
step:1467/1480 train_time:228853ms step_avg:157.07ms
step:1468/1480 train_time:229015ms step_avg:157.08ms
step:1469/1480 train_time:229178ms step_avg:157.08ms
step:1470/1480 train_time:229346ms step_avg:157.09ms
step:1471/1480 train_time:229515ms step_avg:157.09ms
step:1472/1480 train_time:229688ms step_avg:157.11ms
step:1473/1480 train_time:229850ms step_avg:157.11ms
step:1474/1480 train_time:230016ms step_avg:157.11ms
step:1475/1480 train_time:230186ms step_avg:157.12ms
step:1476/1480 train_time:230349ms step_avg:157.13ms
step:1477/1480 train_time:230518ms step_avg:157.14ms
step:1478/1480 train_time:230689ms step_avg:157.15ms
step:1479/1480 train_time:230854ms step_avg:157.15ms
step:1480/1480 train_time:231018ms step_avg:157.15ms
step:1480/1480 val_loss:3.2760 train_time:231093ms step_avg:157.21ms
peak memory consumption: 34239 MiB