"""
train_gpt_simple.py

This file descends from the [NanoGPT speedrun](https://github.com/KellerJordan/modded-nanogpt).
It was prepared as a simplified version of the speedrun for use in neural net optimization research.
"""

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
from pathlib import Path

import torch
from torch import Tensor, nn
from torch.optim import AdamW
import torch.nn.functional as F
import torch.distributed as dist


########################################
#              Dataloader              #
########################################

def _load_data_shard(file: Path):
    header = torch.from_file(str(file), False, 256, dtype=torch.int32) # header is 256 int32
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    num_tokens = int(header[2]) # number of tokens (claimed)
    with file.open("rb", buffering=0) as f:
        tokens = torch.empty(num_tokens, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy()) # avoid bytes->array copy
        assert nbytes == 2 * num_tokens, "number of tokens read does not match header"
    return tokens

def distributed_data_generator(filename_pattern: str, batch_size: int, seq_len=1024):
    files = sorted(Path.cwd().glob(filename_pattern))
    assert batch_size % dist.get_world_size() == 0
    local_batch_size = batch_size // dist.get_world_size()
    file_iter = iter(files)
    tokens, pos = _load_data_shard(next(file_iter)), 0
    while True:
        if pos + batch_size + 1 >= len(tokens):
            tokens, pos = _load_data_shard(next(file_iter)), 0
        buf = tokens[pos + dist.get_rank() * local_batch_size:][:local_batch_size + 1]
        inputs = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True)
        targets = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True)
        pos += batch_size
        yield inputs.view(-1, seq_len), targets.view(-1, seq_len)


########################################
#             Architecture             #
########################################

class RMSNorm(nn.Module):
    def __init__(self, dim):
        super().__init__()
        self.gains = nn.Parameter(torch.ones(dim))

    def forward(self, x):
        return F.rms_norm(x, (x.size(-1),), weight=self.gains.type_as(x))

class Linear(nn.Linear):
    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=True)

    def forward(self, x):
        return F.linear(x, self.weight.type_as(x), self.bias.type_as(x))

class Rotary(nn.Module):
    def __init__(self, dim: int):
        super().__init__()
        # half-truncate RoPE (w/ base freq tuning)
        angular_freq = (1 / 1024) ** torch.linspace(0, 1, steps=dim//4, dtype=torch.float32)
        self.register_buffer("angular_freq", torch.cat([angular_freq, angular_freq.new_zeros(dim//4)]))

    def forward(self, x_BTHD: Tensor):
        pos = torch.arange(x_BTHD.size(1), dtype=torch.float32, device=x_BTHD.device)
        theta = torch.outer(pos, self.angular_freq)[None, :, None, :]
        cos, sin = theta.cos(), theta.sin()
        x1, x2 = x_BTHD.to(dtype=torch.float32).chunk(2, dim=-1)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x_BTHD)

class CausalSelfAttention(nn.Module):
    def __init__(self, dim: int, head_dim=128):
        super().__init__()
        self.num_heads = dim // head_dim
        self.head_dim = head_dim
        hdim = self.num_heads * self.head_dim
        self.q = Linear(dim, hdim)
        self.k = Linear(dim, hdim)
        self.v = Linear(dim, hdim)
        self.proj = Linear(hdim, dim)
        self.rotary = Rotary(head_dim)

    def forward(self, x: Tensor):
        B, T = x.size(0), x.size(1)
        q = self.q(x).view(B, T, self.num_heads, self.head_dim)
        k = self.k(x).view(B, T, self.num_heads, self.head_dim)
        v = self.v(x).view(B, T, self.num_heads, self.head_dim)
        q, k = F.rms_norm(q, (q.size(-1),)), F.rms_norm(k, (k.size(-1),))
        q, k = self.rotary(q), self.rotary(k)
        y = F.scaled_dot_product_attention(q.transpose(1, 2), k.transpose(1, 2),
                                           v.transpose(1, 2), scale=0.12, is_causal=True).transpose(1, 2)
        y = y.contiguous().view(B, T, self.num_heads * self.head_dim)
        y = self.proj(y)
        return y

class MLP(nn.Module):
    def __init__(self, dim: int):
        super().__init__()
        hdim = 4 * dim
        self.fc = Linear(dim, hdim)
        self.proj = Linear(hdim, dim)

    def forward(self, x: Tensor):
        x = self.fc(x)
        x = x.relu().square()
        x = self.proj(x)
        return x

class Block(nn.Module):
    def __init__(self, dim: int):
        super().__init__()
        self.attn = CausalSelfAttention(dim)
        self.mlp = MLP(dim)
        self.norm1 = RMSNorm(dim)
        self.norm2 = RMSNorm(dim)

    def forward(self, x: Tensor):
        x = x + self.attn(self.norm1(x))
        x = x + self.mlp(self.norm2(x))
        return x

class GPT(nn.Module):
    def __init__(self, vocab_size: int, num_layers: int, model_dim: int):
        super().__init__()
        self.embed = nn.Embedding(vocab_size, model_dim).bfloat16()
        self.blocks = nn.ModuleList([Block(model_dim) for _ in range(num_layers)])
        self.proj = Linear(model_dim, vocab_size)
        self.norm1 = RMSNorm(model_dim)
        self.norm2 = RMSNorm(model_dim)

    def forward(self, inputs: Tensor, targets: Tensor):
        x = self.norm1(self.embed(inputs))
        for block in self.blocks:
            x = block(x)
        logits = self.proj(self.norm2(x)).float()
        logits = 15 * logits * (logits.square() + 15**2).rsqrt()
        return F.cross_entropy(logits.view(targets.numel(), -1), targets.view(-1), reduction="sum")


########################################
#              Optimizer               #
########################################

def zeropower_via_newtonschulz5(G: Tensor) -> Tensor:
    assert G.ndim >= 2
    X = G.bfloat16()
    if G.size(-2) > G.size(-1):
        X = X.mT

    # Ensure spectral norm is at most 1
    X = X / (X.norm(dim=(-2, -1), keepdim=True) + 1e-7)
    # Perform the NS iterations, not optimizing for wallclock speed
    a, b, c = 2, -1.5, 0.5
    for _ in range(12):
        A = X @ X.mT
        B = b * A + c * A @ A
        X = a * X + B @ X

    if G.size(-2) > G.size(-1):
        X = X.mT
    return X

@torch.compile
def muon_update(grad, momentum, mu=0.95, nesterov=True):
    momentum.lerp_(grad, 1 - mu)
    update = grad.lerp_(momentum, mu) if nesterov else momentum
    update = zeropower_via_newtonschulz5(update)
    update *= max(1, grad.size(-2) / grad.size(-1))**0.5
    return update

class Muon(torch.optim.Optimizer):
    def __init__(self, params, lr=0.02, weight_decay=0, mu=0.95):
        assert isinstance(params, list) and len(params) >= 1 and isinstance(params[0], torch.nn.Parameter)
        params = sorted(params, key=lambda x: x.size(), reverse=True)
        defaults = dict(lr=lr, weight_decay=weight_decay, mu=mu)
        super().__init__(params, defaults)

    @torch.no_grad()
    def step(self):
        world_size = dist.get_world_size()
        rank = dist.get_rank()
        for group in self.param_groups:
            params = group["params"]
            params_pad = params + [torch.empty_like(params[-1])] * (world_size - len(params) % world_size)
            for base_i in range(0, len(params), world_size):
                if base_i + rank < len(params):
                    p = params[base_i + rank]
                    state = self.state[p]
                    if len(state) == 0:
                        state["momentum"] = torch.zeros_like(p)
                    update = muon_update(p.grad, state["momentum"], mu=group["mu"])
                    p.mul_(1 - group["lr"] * group["weight_decay"])
                    p.add_(update, alpha=-group["lr"])
                dist.all_gather(params_pad[base_i:base_i + world_size], params_pad[base_i + rank])


########################################
#                Setup                 #
########################################

# torchrun sets these env variables
device = torch.device("cuda", int(os.environ["LOCAL_RANK"]))
torch.cuda.set_device(device)
dist.init_process_group(backend="nccl", device_id=device)
dist.barrier()
# this code can be run equivalently with 1, 2, 4, or 8 gpus.
assert 8 % dist.get_world_size() == 0

# logging setup
if dist.get_rank() == 0:
    os.makedirs("logs", exist_ok=True)
    logfile = f"logs/{uuid.uuid4()}.txt"
    print(logfile)
def print0(s, console=False, log=True):
    if dist.get_rank() == 0:
        if console:
            print(s)
        if log:
            with open(logfile, "a") as f:
                print(s, file=f)

# we begin by logging this file itself
print0(code)
print0("="*100)
print0(f"Running PyTorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}"
       + f" on {torch.cuda.get_device_name(device)} with world_size {dist.get_world_size()}")
print0("="*100)

val_tokens = 20 * 524288
batch_size = 8 * 64 * 1024
mbs = 64
val_inputs, val_targets = next(distributed_data_generator("data/fineweb10B/fineweb_val_*.bin", val_tokens))

model = GPT(vocab_size=50304, num_layers=12, model_dim=768).cuda()
model.compile(dynamic=False)


num_trials = int(sys.argv[-1]) if len(sys.argv) > 1 else 1

for _ in range(num_trials):


    ########################################
    #       Init & Optim Hyperparams       #
    ########################################

    # we want to minimize this while still reaching 3.28 val loss
    train_steps = 3350

    # initialize model parameters
    for name, p in model.named_parameters():
        w = p.data
        if name.endswith("weight"):
            if "proj" in name:
                w.zero_()
            elif "embed" in name:
                w.normal_()  # default torch init
            else:
                w.normal_(std=0.33**0.5 / w.size(-1)**0.5)  # default torch init
        elif name.endswith("bias"):
            w.zero_()
        elif name.endswith("gains"):
            w.normal_(mean=1, std=0)
        else:
            raise Exception(f"Uninitialized parameter: {name}")

    # create the optimizer(s)
    optimizer1 = AdamW([dict(params=[model.embed.weight], lr=0.3),
                        dict(params=[model.proj.weight], lr=1/320),
                        dict(params=[p for p in model.parameters() if p.ndim < 2], lr=0.01)],
                       betas=(0.8, 0.95), eps=1e-10, weight_decay=0, fused=True)
    optimizer2 = Muon([p for p in model.blocks.parameters() if p.ndim >= 2],
                      lr=0.035, weight_decay=0.025)
    optimizers = [optimizer1, optimizer2]
    assert set(p for opt in optimizers for group in opt.param_groups
               for p in group["params"]) == set(model.parameters())
    for opt in optimizers:
        for group in opt.param_groups:
            group["initial_lr"] = group["lr"]

    # learning rate schedule: stable then decay
    def set_hparams(step, cooldown_frac=0.7):
        progress = step / train_steps
        assert 0 <= progress < 1
        if progress < 1 - cooldown_frac:
            eta = 1.0
        else:
            eta = (1 - progress) / cooldown_frac
        for opt in optimizers:
            for group in opt.param_groups:
                group["lr"] = group["initial_lr"] * eta


    ########################################
    #        Training and Validation       #
    ########################################

    train_loader = distributed_data_generator("data/fineweb10B/fineweb_train_*.bin", batch_size)
    for p in model.parameters():
        dist.broadcast(p.detach(), 0)
    # start the clock
    training_time = 0
    last_val_step = 0
    dist.barrier()
    t0 = time.perf_counter()
    for step in range(train_steps + 1):

        # --------------- VALIDATION SECTION -----------------
        val_step_freq = 125 if step / train_steps < 0.9 else 25
        if step == train_steps or step % val_step_freq == 0:
            # stop the clock
            dist.barrier()
            time_since_last_val = time.perf_counter() - t0
            step_avg = time_since_last_val / (step - last_val_step) if step > 0 else float("nan")
            last_val_step = step
            training_time += time_since_last_val
            model.eval()
            val_loss = 0
            with torch.no_grad():
                assert len(val_inputs) % mbs == 0
                for i in range(len(val_inputs) // mbs):
                    val_loss += model(val_inputs[i*mbs:(i+1)*mbs], val_targets[i*mbs:(i+1)*mbs])
            dist.all_reduce(val_loss, op=dist.ReduceOp.SUM)
            val_loss /= val_tokens
            print0(f"step:{step}/{train_steps} val_loss:{val_loss:.5f} train_time:{training_time:.3f}s"
                   + f" step_avg:{1000*step_avg:.2f}ms", console=True)
            model.train()
            # start the clock again
            dist.barrier()
            t0 = time.perf_counter()

        if step == train_steps:
            break

        # --------------- TRAINING SECTION -----------------
        inputs, targets = next(train_loader)
        # accumulate across microbatches in case we are running with fewer than 8 gpus
        assert len(inputs) % mbs == 0
        for i in range(len(inputs) // mbs):
            model(inputs[i*mbs:(i+1)*mbs], targets[i*mbs:(i+1)*mbs]).backward()
        for name, p in model.named_parameters():
            assert p.grad is not None, name
            dist.all_reduce(p.grad, op=dist.ReduceOp.SUM)
        # set optimization hyperparameters and take a step
        set_hparams(step)
        for opt in optimizers:
            opt.step()
        model.zero_grad(set_to_none=True)
        approx_training_time = training_time + (time.perf_counter() - t0)
        print0(f"step:{step+1}/{train_steps} train_time:{approx_training_time:.3f}s"
               + f" step_avg:{1000*approx_training_time/(step + 1):.2f}ms", console=True, log=False)

dist.destroy_process_group()

====================================================================================================
Running PyTorch 2.11.0+cu128 compiled for CUDA 12.8 on NVIDIA H100 80GB HBM3 with world_size 8
====================================================================================================
step:0/3350 val_loss:10.82584 train_time:0.001s step_avg:nanms
step:125/3350 val_loss:4.66393 train_time:22.870s step_avg:182.95ms
step:250/3350 val_loss:4.12440 train_time:41.354s step_avg:147.87ms
step:375/3350 val_loss:3.93798 train_time:59.719s step_avg:146.92ms
step:500/3350 val_loss:3.83472 train_time:78.115s step_avg:147.17ms
step:625/3350 val_loss:3.76462 train_time:96.500s step_avg:147.08ms
step:750/3350 val_loss:3.71744 train_time:114.861s step_avg:146.89ms
step:875/3350 val_loss:3.67776 train_time:133.226s step_avg:146.92ms
step:1000/3350 val_loss:3.64527 train_time:151.604s step_avg:147.03ms
step:1125/3350 val_loss:3.61513 train_time:169.962s step_avg:146.86ms
step:1250/3350 val_loss:3.58228 train_time:188.331s step_avg:146.96ms
step:1375/3350 val_loss:3.55712 train_time:206.692s step_avg:146.89ms
step:1500/3350 val_loss:3.52950 train_time:225.041s step_avg:146.79ms
step:1625/3350 val_loss:3.50900 train_time:243.399s step_avg:146.87ms
step:1750/3350 val_loss:3.48547 train_time:261.770s step_avg:146.96ms
step:1875/3350 val_loss:3.46476 train_time:280.169s step_avg:147.20ms
step:2000/3350 val_loss:3.44642 train_time:298.532s step_avg:146.90ms
step:2125/3350 val_loss:3.42888 train_time:316.910s step_avg:147.02ms
step:2250/3350 val_loss:3.41061 train_time:335.252s step_avg:146.73ms
step:2375/3350 val_loss:3.39424 train_time:353.622s step_avg:146.96ms
step:2500/3350 val_loss:3.37751 train_time:371.992s step_avg:146.96ms
step:2625/3350 val_loss:3.36031 train_time:390.342s step_avg:146.80ms
step:2750/3350 val_loss:3.34392 train_time:408.707s step_avg:146.92ms
step:2875/3350 val_loss:3.32812 train_time:427.076s step_avg:146.95ms
step:3000/3350 val_loss:3.31193 train_time:445.414s step_avg:146.71ms
step:3025/3350 val_loss:3.30810 train_time:449.090s step_avg:147.02ms
step:3050/3350 val_loss:3.30500 train_time:452.780s step_avg:147.62ms
step:3075/3350 val_loss:3.30216 train_time:456.455s step_avg:147.00ms
step:3100/3350 val_loss:3.29931 train_time:460.123s step_avg:146.73ms
step:3125/3350 val_loss:3.29650 train_time:463.794s step_avg:146.81ms
step:3150/3350 val_loss:3.29350 train_time:467.466s step_avg:146.88ms
step:3175/3350 val_loss:3.29090 train_time:471.140s step_avg:146.98ms
step:3200/3350 val_loss:3.28841 train_time:474.812s step_avg:146.87ms
step:3225/3350 val_loss:3.28604 train_time:478.483s step_avg:146.82ms
step:3250/3350 val_loss:3.28396 train_time:482.174s step_avg:147.65ms
step:3275/3350 val_loss:3.28209 train_time:485.842s step_avg:146.73ms
step:3300/3350 val_loss:3.28051 train_time:489.516s step_avg:146.95ms
step:3325/3350 val_loss:3.27931 train_time:493.185s step_avg:146.78ms
step:3350/3350 val_loss:3.27874 train_time:496.855s step_avg:146.79ms
step:0/3350 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3350 val_loss:4.69112 train_time:18.442s step_avg:147.54ms
step:250/3350 val_loss:4.13775 train_time:36.840s step_avg:147.19ms
step:375/3350 val_loss:3.94004 train_time:55.212s step_avg:146.97ms
step:500/3350 val_loss:3.83419 train_time:73.588s step_avg:147.00ms
step:625/3350 val_loss:3.76874 train_time:91.975s step_avg:147.10ms
step:750/3350 val_loss:3.71632 train_time:110.336s step_avg:146.89ms
step:875/3350 val_loss:3.67672 train_time:128.705s step_avg:146.95ms
step:1000/3350 val_loss:3.63967 train_time:147.085s step_avg:147.04ms
step:1125/3350 val_loss:3.61406 train_time:165.436s step_avg:146.81ms
step:1250/3350 val_loss:3.58036 train_time:183.801s step_avg:146.92ms
step:1375/3350 val_loss:3.55474 train_time:202.169s step_avg:146.94ms
step:1500/3350 val_loss:3.52718 train_time:220.507s step_avg:146.71ms
step:1625/3350 val_loss:3.50738 train_time:238.865s step_avg:146.86ms
step:1750/3350 val_loss:3.48381 train_time:257.235s step_avg:146.96ms
step:1875/3350 val_loss:3.46265 train_time:275.578s step_avg:146.75ms
step:2000/3350 val_loss:3.44451 train_time:293.942s step_avg:146.91ms
step:2125/3350 val_loss:3.42654 train_time:312.309s step_avg:146.94ms
step:2250/3350 val_loss:3.40871 train_time:330.664s step_avg:146.84ms
step:2375/3350 val_loss:3.39231 train_time:349.105s step_avg:147.52ms
step:2500/3350 val_loss:3.37559 train_time:367.471s step_avg:146.93ms
step:2625/3350 val_loss:3.35814 train_time:385.831s step_avg:146.89ms
step:2750/3350 val_loss:3.34188 train_time:404.195s step_avg:146.91ms
step:2875/3350 val_loss:3.32554 train_time:422.571s step_avg:147.00ms
step:3000/3350 val_loss:3.30982 train_time:440.926s step_avg:146.84ms
step:3025/3350 val_loss:3.30570 train_time:444.602s step_avg:147.01ms
step:3050/3350 val_loss:3.30275 train_time:448.294s step_avg:147.71ms
step:3075/3350 val_loss:3.29975 train_time:451.970s step_avg:147.02ms
step:3100/3350 val_loss:3.29688 train_time:455.646s step_avg:147.05ms
step:3125/3350 val_loss:3.29421 train_time:459.320s step_avg:146.96ms
step:3150/3350 val_loss:3.29112 train_time:462.995s step_avg:147.00ms
step:3175/3350 val_loss:3.28852 train_time:466.665s step_avg:146.80ms
step:3200/3350 val_loss:3.28599 train_time:470.340s step_avg:146.99ms
step:3225/3350 val_loss:3.28367 train_time:474.010s step_avg:146.78ms
step:3250/3350 val_loss:3.28154 train_time:477.702s step_avg:147.68ms
step:3275/3350 val_loss:3.27975 train_time:481.373s step_avg:146.85ms
step:3300/3350 val_loss:3.27818 train_time:485.043s step_avg:146.79ms
step:3325/3350 val_loss:3.27699 train_time:488.716s step_avg:146.94ms
step:3350/3350 val_loss:3.27640 train_time:492.390s step_avg:146.95ms
step:0/3350 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3350 val_loss:4.68014 train_time:18.457s step_avg:147.66ms
step:250/3350 val_loss:4.12173 train_time:36.864s step_avg:147.26ms
step:375/3350 val_loss:3.94005 train_time:55.257s step_avg:147.14ms
step:500/3350 val_loss:3.83722 train_time:73.666s step_avg:147.27ms
step:625/3350 val_loss:3.76686 train_time:92.054s step_avg:147.10ms
step:750/3350 val_loss:3.71897 train_time:110.417s step_avg:146.91ms
step:875/3350 val_loss:3.67812 train_time:128.806s step_avg:147.11ms
step:1000/3350 val_loss:3.64388 train_time:147.199s step_avg:147.14ms
step:1125/3350 val_loss:3.61527 train_time:165.548s step_avg:146.79ms
step:1250/3350 val_loss:3.58143 train_time:183.923s step_avg:147.01ms
step:1375/3350 val_loss:3.55519 train_time:202.297s step_avg:146.99ms
step:1500/3350 val_loss:3.52868 train_time:220.673s step_avg:147.01ms
step:1625/3350 val_loss:3.50889 train_time:239.056s step_avg:147.06ms
step:1750/3350 val_loss:3.48532 train_time:257.432s step_avg:147.00ms
step:1875/3350 val_loss:3.46509 train_time:275.798s step_avg:146.93ms
step:2000/3350 val_loss:3.44501 train_time:294.170s step_avg:146.97ms
step:2125/3350 val_loss:3.42769 train_time:312.540s step_avg:146.96ms
step:2250/3350 val_loss:3.41004 train_time:330.900s step_avg:146.88ms
step:2375/3350 val_loss:3.39390 train_time:349.259s step_avg:146.88ms
step:2500/3350 val_loss:3.37668 train_time:367.644s step_avg:147.08ms
step:2625/3350 val_loss:3.35952 train_time:385.998s step_avg:146.83ms
step:2750/3350 val_loss:3.34330 train_time:404.379s step_avg:147.05ms
step:2875/3350 val_loss:3.32695 train_time:422.792s step_avg:147.31ms
step:3000/3350 val_loss:3.31092 train_time:441.142s step_avg:146.80ms
step:3025/3350 val_loss:3.30709 train_time:444.815s step_avg:146.92ms
step:3050/3350 val_loss:3.30400 train_time:448.507s step_avg:147.68ms
step:3075/3350 val_loss:3.30120 train_time:452.186s step_avg:147.15ms
step:3100/3350 val_loss:3.29821 train_time:455.856s step_avg:146.80ms
step:3125/3350 val_loss:3.29545 train_time:459.534s step_avg:147.10ms
step:3150/3350 val_loss:3.29240 train_time:463.214s step_avg:147.21ms
step:3175/3350 val_loss:3.28972 train_time:466.892s step_avg:147.12ms
step:3200/3350 val_loss:3.28708 train_time:470.570s step_avg:147.14ms
step:3225/3350 val_loss:3.28484 train_time:474.242s step_avg:146.89ms
step:3250/3350 val_loss:3.28278 train_time:477.936s step_avg:147.73ms
step:3275/3350 val_loss:3.28097 train_time:481.612s step_avg:147.04ms
step:3300/3350 val_loss:3.27940 train_time:485.286s step_avg:146.96ms
step:3325/3350 val_loss:3.27821 train_time:488.958s step_avg:146.90ms
step:3350/3350 val_loss:3.27764 train_time:492.633s step_avg:146.98ms
step:0/3350 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3350 val_loss:4.67289 train_time:18.445s step_avg:147.56ms
step:250/3350 val_loss:4.12426 train_time:36.850s step_avg:147.24ms
step:375/3350 val_loss:3.94035 train_time:55.238s step_avg:147.11ms
step:500/3350 val_loss:3.84059 train_time:73.625s step_avg:147.10ms
step:625/3350 val_loss:3.76570 train_time:92.016s step_avg:147.13ms
step:750/3350 val_loss:3.71972 train_time:110.392s step_avg:147.01ms
step:875/3350 val_loss:3.67893 train_time:128.776s step_avg:147.07ms
step:1000/3350 val_loss:3.64702 train_time:147.162s step_avg:147.09ms
step:1125/3350 val_loss:3.61549 train_time:165.516s step_avg:146.83ms
step:1250/3350 val_loss:3.58333 train_time:183.890s step_avg:146.99ms
step:1375/3350 val_loss:3.55672 train_time:202.253s step_avg:146.90ms
step:1500/3350 val_loss:3.53134 train_time:220.605s step_avg:146.82ms
step:1625/3350 val_loss:3.50898 train_time:238.974s step_avg:146.95ms
step:1750/3350 val_loss:3.48665 train_time:257.345s step_avg:146.97ms
step:1875/3350 val_loss:3.46793 train_time:275.707s step_avg:146.90ms
step:2000/3350 val_loss:3.44745 train_time:294.072s step_avg:146.92ms
step:2125/3350 val_loss:3.42947 train_time:312.454s step_avg:147.05ms
step:2250/3350 val_loss:3.41211 train_time:330.806s step_avg:146.81ms
step:2375/3350 val_loss:3.39547 train_time:349.178s step_avg:146.98ms
step:2500/3350 val_loss:3.37906 train_time:367.545s step_avg:146.94ms
step:2625/3350 val_loss:3.36176 train_time:385.891s step_avg:146.76ms
step:2750/3350 val_loss:3.34510 train_time:404.266s step_avg:147.01ms
step:2875/3350 val_loss:3.32913 train_time:422.649s step_avg:147.06ms
step:3000/3350 val_loss:3.31306 train_time:441.005s step_avg:146.85ms
step:3025/3350 val_loss:3.30928 train_time:444.678s step_avg:146.92ms
step:3050/3350 val_loss:3.30619 train_time:448.373s step_avg:147.79ms
step:3075/3350 val_loss:3.30336 train_time:452.043s step_avg:146.82ms
step:3100/3350 val_loss:3.30050 train_time:455.712s step_avg:146.75ms
step:3125/3350 val_loss:3.29775 train_time:459.386s step_avg:146.94ms
step:3150/3350 val_loss:3.29472 train_time:463.066s step_avg:147.22ms
step:3175/3350 val_loss:3.29212 train_time:466.739s step_avg:146.91ms
step:3200/3350 val_loss:3.28958 train_time:470.484s step_avg:149.81ms
step:3225/3350 val_loss:3.28724 train_time:474.157s step_avg:146.94ms
step:3250/3350 val_loss:3.28512 train_time:477.851s step_avg:147.74ms
step:3275/3350 val_loss:3.28326 train_time:481.523s step_avg:146.89ms
step:3300/3350 val_loss:3.28174 train_time:485.193s step_avg:146.78ms
step:3325/3350 val_loss:3.28056 train_time:488.865s step_avg:146.90ms
step:3350/3350 val_loss:3.27999 train_time:492.541s step_avg:147.03ms
step:0/3350 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3350 val_loss:4.67134 train_time:18.456s step_avg:147.65ms
step:250/3350 val_loss:4.12477 train_time:36.877s step_avg:147.37ms
step:375/3350 val_loss:3.93743 train_time:55.267s step_avg:147.12ms
step:500/3350 val_loss:3.83492 train_time:73.661s step_avg:147.15ms
step:625/3350 val_loss:3.76661 train_time:92.048s step_avg:147.10ms
step:750/3350 val_loss:3.71718 train_time:110.410s step_avg:146.89ms
step:875/3350 val_loss:3.67636 train_time:128.787s step_avg:147.02ms
step:1000/3350 val_loss:3.64203 train_time:147.167s step_avg:147.04ms
step:1125/3350 val_loss:3.61388 train_time:165.510s step_avg:146.74ms
step:1250/3350 val_loss:3.58163 train_time:183.884s step_avg:146.99ms
step:1375/3350 val_loss:3.55410 train_time:202.249s step_avg:146.92ms
step:1500/3350 val_loss:3.52891 train_time:220.603s step_avg:146.83ms
step:1625/3350 val_loss:3.50804 train_time:238.982s step_avg:147.03ms
step:1750/3350 val_loss:3.48497 train_time:257.349s step_avg:146.94ms
step:1875/3350 val_loss:3.46464 train_time:275.692s step_avg:146.74ms
step:2000/3350 val_loss:3.44532 train_time:294.059s step_avg:146.94ms
step:2125/3350 val_loss:3.42733 train_time:312.440s step_avg:147.05ms
step:2250/3350 val_loss:3.40936 train_time:330.789s step_avg:146.79ms
step:2375/3350 val_loss:3.39297 train_time:349.159s step_avg:146.97ms
step:2500/3350 val_loss:3.37639 train_time:367.547s step_avg:147.10ms
step:2625/3350 val_loss:3.35898 train_time:385.903s step_avg:146.85ms
step:2750/3350 val_loss:3.34295 train_time:404.273s step_avg:146.96ms
step:2875/3350 val_loss:3.32669 train_time:422.632s step_avg:146.87ms
step:3000/3350 val_loss:3.31100 train_time:440.989s step_avg:146.86ms
step:3025/3350 val_loss:3.30705 train_time:444.662s step_avg:146.94ms
step:3050/3350 val_loss:3.30417 train_time:448.354s step_avg:147.66ms
step:3075/3350 val_loss:3.30126 train_time:452.029s step_avg:147.02ms
step:3100/3350 val_loss:3.29827 train_time:455.705s step_avg:147.03ms
step:3125/3350 val_loss:3.29559 train_time:459.385s step_avg:147.20ms
step:3150/3350 val_loss:3.29237 train_time:463.058s step_avg:146.93ms
step:3175/3350 val_loss:3.28983 train_time:466.731s step_avg:146.92ms
step:3200/3350 val_loss:3.28728 train_time:470.404s step_avg:146.92ms
step:3225/3350 val_loss:3.28496 train_time:474.080s step_avg:147.00ms
step:3250/3350 val_loss:3.28285 train_time:477.770s step_avg:147.64ms
step:3275/3350 val_loss:3.28098 train_time:481.447s step_avg:147.05ms
step:3300/3350 val_loss:3.27941 train_time:485.118s step_avg:146.85ms
step:3325/3350 val_loss:3.27824 train_time:488.790s step_avg:146.86ms
step:3350/3350 val_loss:3.27765 train_time:492.459s step_avg:146.79ms
step:0/3350 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3350 val_loss:4.67019 train_time:18.450s step_avg:147.60ms
step:250/3350 val_loss:4.12446 train_time:36.875s step_avg:147.39ms
step:375/3350 val_loss:3.93960 train_time:55.317s step_avg:147.54ms
step:500/3350 val_loss:3.83459 train_time:73.715s step_avg:147.18ms
step:625/3350 val_loss:3.76735 train_time:92.116s step_avg:147.20ms
step:750/3350 val_loss:3.71877 train_time:110.483s step_avg:146.94ms
step:875/3350 val_loss:3.67913 train_time:128.877s step_avg:147.16ms
step:1000/3350 val_loss:3.64358 train_time:147.259s step_avg:147.05ms
step:1125/3350 val_loss:3.61462 train_time:165.629s step_avg:146.96ms
step:1250/3350 val_loss:3.58258 train_time:184.020s step_avg:147.13ms
step:1375/3350 val_loss:3.55581 train_time:202.403s step_avg:147.07ms
step:1500/3350 val_loss:3.52742 train_time:220.753s step_avg:146.80ms
step:1625/3350 val_loss:3.50893 train_time:239.139s step_avg:147.09ms
step:1750/3350 val_loss:3.48495 train_time:257.523s step_avg:147.07ms
step:1875/3350 val_loss:3.46537 train_time:275.880s step_avg:146.86ms
step:2000/3350 val_loss:3.44547 train_time:294.255s step_avg:146.99ms
step:2125/3350 val_loss:3.42823 train_time:312.620s step_avg:146.92ms
step:2250/3350 val_loss:3.41030 train_time:330.990s step_avg:146.96ms
step:2375/3350 val_loss:3.39394 train_time:349.362s step_avg:146.97ms
step:2500/3350 val_loss:3.37751 train_time:367.747s step_avg:147.08ms
step:2625/3350 val_loss:3.36015 train_time:386.115s step_avg:146.95ms
step:2750/3350 val_loss:3.34365 train_time:404.499s step_avg:147.08ms
step:2875/3350 val_loss:3.32772 train_time:422.886s step_avg:147.10ms
step:3000/3350 val_loss:3.31179 train_time:441.255s step_avg:146.95ms
step:3025/3350 val_loss:3.30800 train_time:444.931s step_avg:147.04ms
step:3050/3350 val_loss:3.30498 train_time:448.627s step_avg:147.84ms
step:3075/3350 val_loss:3.30208 train_time:452.301s step_avg:146.96ms
step:3100/3350 val_loss:3.29919 train_time:455.979s step_avg:147.12ms
step:3125/3350 val_loss:3.29660 train_time:459.655s step_avg:147.01ms
step:3150/3350 val_loss:3.29350 train_time:463.334s step_avg:147.17ms
step:3175/3350 val_loss:3.29085 train_time:467.008s step_avg:146.95ms
step:3200/3350 val_loss:3.28834 train_time:470.685s step_avg:147.09ms
step:3225/3350 val_loss:3.28601 train_time:474.360s step_avg:146.97ms
step:3250/3350 val_loss:3.28401 train_time:478.059s step_avg:147.99ms
step:3275/3350 val_loss:3.28219 train_time:481.734s step_avg:146.99ms
step:3300/3350 val_loss:3.28059 train_time:485.414s step_avg:147.19ms
step:3325/3350 val_loss:3.27939 train_time:489.091s step_avg:147.10ms
step:3350/3350 val_loss:3.27879 train_time:492.766s step_avg:147.00ms
step:0/3350 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3350 val_loss:4.68702 train_time:18.451s step_avg:147.61ms
step:250/3350 val_loss:4.13694 train_time:36.873s step_avg:147.38ms
step:375/3350 val_loss:3.94563 train_time:55.271s step_avg:147.18ms
step:500/3350 val_loss:3.83729 train_time:73.677s step_avg:147.25ms
step:625/3350 val_loss:3.76802 train_time:92.078s step_avg:147.21ms
step:750/3350 val_loss:3.71752 train_time:110.525s step_avg:147.57ms
step:875/3350 val_loss:3.67817 train_time:128.913s step_avg:147.11ms
step:1000/3350 val_loss:3.64294 train_time:147.294s step_avg:147.05ms
step:1125/3350 val_loss:3.61555 train_time:165.663s step_avg:146.95ms
step:1250/3350 val_loss:3.58221 train_time:184.047s step_avg:147.07ms
step:1375/3350 val_loss:3.55627 train_time:202.425s step_avg:147.02ms
step:1500/3350 val_loss:3.52837 train_time:220.789s step_avg:146.91ms
step:1625/3350 val_loss:3.50884 train_time:239.162s step_avg:146.99ms
step:1750/3350 val_loss:3.48517 train_time:257.540s step_avg:147.02ms
step:1875/3350 val_loss:3.46520 train_time:275.911s step_avg:146.97ms
step:2000/3350 val_loss:3.44607 train_time:294.294s step_avg:147.07ms
step:2125/3350 val_loss:3.42795 train_time:312.662s step_avg:146.94ms
step:2250/3350 val_loss:3.41072 train_time:331.043s step_avg:147.05ms
step:2375/3350 val_loss:3.39449 train_time:349.413s step_avg:146.96ms
step:2500/3350 val_loss:3.37766 train_time:367.793s step_avg:147.05ms
step:2625/3350 val_loss:3.36010 train_time:386.149s step_avg:146.84ms
step:2750/3350 val_loss:3.34388 train_time:404.534s step_avg:147.08ms
step:2875/3350 val_loss:3.32766 train_time:422.908s step_avg:146.99ms
step:3000/3350 val_loss:3.31171 train_time:441.273s step_avg:146.92ms
step:3025/3350 val_loss:3.30781 train_time:444.944s step_avg:146.85ms
step:3050/3350 val_loss:3.30479 train_time:448.638s step_avg:147.73ms
step:3075/3350 val_loss:3.30194 train_time:452.313s step_avg:147.00ms
step:3100/3350 val_loss:3.29907 train_time:455.987s step_avg:147.00ms
step:3125/3350 val_loss:3.29652 train_time:459.665s step_avg:147.12ms
step:3150/3350 val_loss:3.29336 train_time:463.345s step_avg:147.18ms
step:3175/3350 val_loss:3.29084 train_time:467.018s step_avg:146.94ms
step:3200/3350 val_loss:3.28819 train_time:470.698s step_avg:147.19ms
step:3225/3350 val_loss:3.28591 train_time:474.371s step_avg:146.92ms
step:3250/3350 val_loss:3.28380 train_time:478.062s step_avg:147.61ms
step:3275/3350 val_loss:3.28195 train_time:481.738s step_avg:147.07ms
step:3300/3350 val_loss:3.28040 train_time:485.415s step_avg:147.08ms
step:3325/3350 val_loss:3.27919 train_time:489.089s step_avg:146.93ms
step:3350/3350 val_loss:3.27861 train_time:492.769s step_avg:147.22ms
step:0/3350 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3350 val_loss:4.66812 train_time:18.435s step_avg:147.48ms
step:250/3350 val_loss:4.12936 train_time:36.866s step_avg:147.45ms
step:375/3350 val_loss:3.94019 train_time:55.256s step_avg:147.12ms
step:500/3350 val_loss:3.83802 train_time:73.654s step_avg:147.19ms
step:625/3350 val_loss:3.76704 train_time:92.060s step_avg:147.24ms
step:750/3350 val_loss:3.72027 train_time:110.427s step_avg:146.94ms
step:875/3350 val_loss:3.67728 train_time:128.813s step_avg:147.08ms
step:1000/3350 val_loss:3.64119 train_time:147.215s step_avg:147.22ms
step:1125/3350 val_loss:3.61617 train_time:165.600s step_avg:147.08ms
step:1250/3350 val_loss:3.58322 train_time:184.076s step_avg:147.80ms
step:1375/3350 val_loss:3.55633 train_time:202.464s step_avg:147.11ms
step:1500/3350 val_loss:3.53067 train_time:220.843s step_avg:147.03ms
step:1625/3350 val_loss:3.50888 train_time:239.220s step_avg:147.02ms
step:1750/3350 val_loss:3.48597 train_time:257.605s step_avg:147.08ms
step:1875/3350 val_loss:3.46566 train_time:275.977s step_avg:146.98ms
step:2000/3350 val_loss:3.44617 train_time:294.355s step_avg:147.02ms
step:2125/3350 val_loss:3.42869 train_time:312.740s step_avg:147.08ms
step:2250/3350 val_loss:3.41118 train_time:331.104s step_avg:146.91ms
step:2375/3350 val_loss:3.39463 train_time:349.485s step_avg:147.04ms
step:2500/3350 val_loss:3.37816 train_time:367.877s step_avg:147.14ms
step:2625/3350 val_loss:3.36048 train_time:386.258s step_avg:147.04ms
step:2750/3350 val_loss:3.34387 train_time:404.645s step_avg:147.10ms
step:2875/3350 val_loss:3.32807 train_time:423.022s step_avg:147.02ms
step:3000/3350 val_loss:3.31203 train_time:441.396s step_avg:147.00ms
step:3025/3350 val_loss:3.30824 train_time:445.073s step_avg:147.05ms
step:3050/3350 val_loss:3.30520 train_time:448.765s step_avg:147.70ms
step:3075/3350 val_loss:3.30233 train_time:452.440s step_avg:146.99ms
step:3100/3350 val_loss:3.29937 train_time:456.114s step_avg:146.98ms
step:3125/3350 val_loss:3.29679 train_time:459.791s step_avg:147.05ms
step:3150/3350 val_loss:3.29362 train_time:463.469s step_avg:147.15ms
step:3175/3350 val_loss:3.29103 train_time:467.140s step_avg:146.83ms
step:3200/3350 val_loss:3.28841 train_time:470.813s step_avg:146.90ms
step:3225/3350 val_loss:3.28608 train_time:474.490s step_avg:147.11ms
step:3250/3350 val_loss:3.28405 train_time:478.192s step_avg:148.06ms
step:3275/3350 val_loss:3.28221 train_time:481.870s step_avg:147.12ms
step:3300/3350 val_loss:3.28063 train_time:485.547s step_avg:147.08ms
step:3325/3350 val_loss:3.27948 train_time:489.225s step_avg:147.11ms
step:3350/3350 val_loss:3.27889 train_time:492.901s step_avg:147.04ms
step:0/3350 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3350 val_loss:4.67305 train_time:18.459s step_avg:147.67ms
step:250/3350 val_loss:4.13053 train_time:36.891s step_avg:147.45ms
step:375/3350 val_loss:3.93914 train_time:55.303s step_avg:147.30ms
step:500/3350 val_loss:3.83516 train_time:73.730s step_avg:147.42ms
step:625/3350 val_loss:3.76478 train_time:92.128s step_avg:147.18ms
step:750/3350 val_loss:3.72046 train_time:110.506s step_avg:147.03ms
step:875/3350 val_loss:3.67674 train_time:128.906s step_avg:147.20ms
step:1000/3350 val_loss:3.64314 train_time:147.301s step_avg:147.16ms
step:1125/3350 val_loss:3.61196 train_time:165.682s step_avg:147.04ms
step:1250/3350 val_loss:3.57969 train_time:184.081s step_avg:147.19ms
step:1375/3350 val_loss:3.55507 train_time:202.463s step_avg:147.06ms
step:1500/3350 val_loss:3.52775 train_time:220.837s step_avg:146.99ms
step:1625/3350 val_loss:3.50791 train_time:239.229s step_avg:147.14ms
step:1750/3350 val_loss:3.48601 train_time:257.692s step_avg:147.70ms
step:1875/3350 val_loss:3.46490 train_time:276.044s step_avg:146.82ms
step:2000/3350 val_loss:3.44566 train_time:294.421s step_avg:147.02ms
step:2125/3350 val_loss:3.42726 train_time:312.794s step_avg:146.98ms
step:2250/3350 val_loss:3.41019 train_time:331.155s step_avg:146.89ms
step:2375/3350 val_loss:3.39376 train_time:349.541s step_avg:147.09ms
step:2500/3350 val_loss:3.37729 train_time:367.928s step_avg:147.09ms
step:2625/3350 val_loss:3.35981 train_time:386.298s step_avg:146.96ms
step:2750/3350 val_loss:3.34346 train_time:404.697s step_avg:147.19ms
step:2875/3350 val_loss:3.32761 train_time:423.078s step_avg:147.05ms
step:3000/3350 val_loss:3.31166 train_time:441.452s step_avg:147.00ms
step:3025/3350 val_loss:3.30774 train_time:445.123s step_avg:146.83ms
step:3050/3350 val_loss:3.30468 train_time:448.820s step_avg:147.89ms
step:3075/3350 val_loss:3.30181 train_time:452.496s step_avg:147.04ms
step:3100/3350 val_loss:3.29885 train_time:456.170s step_avg:146.96ms
step:3125/3350 val_loss:3.29607 train_time:459.858s step_avg:147.52ms
step:3150/3350 val_loss:3.29306 train_time:463.538s step_avg:147.21ms
step:3175/3350 val_loss:3.29052 train_time:467.215s step_avg:147.06ms
step:3200/3350 val_loss:3.28792 train_time:470.895s step_avg:147.21ms
step:3225/3350 val_loss:3.28561 train_time:474.571s step_avg:147.06ms
step:3250/3350 val_loss:3.28358 train_time:478.265s step_avg:147.77ms
step:3275/3350 val_loss:3.28175 train_time:481.936s step_avg:146.83ms
step:3300/3350 val_loss:3.28013 train_time:485.613s step_avg:147.07ms
step:3325/3350 val_loss:3.27899 train_time:489.292s step_avg:147.15ms
step:3350/3350 val_loss:3.27842 train_time:492.973s step_avg:147.23ms
step:0/3350 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3350 val_loss:4.69656 train_time:18.459s step_avg:147.67ms
step:250/3350 val_loss:4.14124 train_time:36.893s step_avg:147.47ms
step:375/3350 val_loss:3.94532 train_time:55.282s step_avg:147.11ms
step:500/3350 val_loss:3.83901 train_time:73.696s step_avg:147.31ms
step:625/3350 val_loss:3.76898 train_time:92.100s step_avg:147.24ms
step:750/3350 val_loss:3.71870 train_time:110.482s step_avg:147.06ms
step:875/3350 val_loss:3.67966 train_time:128.882s step_avg:147.20ms
step:1000/3350 val_loss:3.64580 train_time:147.284s step_avg:147.22ms
step:1125/3350 val_loss:3.61563 train_time:165.659s step_avg:147.00ms
step:1250/3350 val_loss:3.58232 train_time:184.048s step_avg:147.11ms
step:1375/3350 val_loss:3.55626 train_time:202.447s step_avg:147.19ms
step:1500/3350 val_loss:3.52871 train_time:220.834s step_avg:147.09ms
step:1625/3350 val_loss:3.50892 train_time:239.221s step_avg:147.10ms
step:1750/3350 val_loss:3.48579 train_time:257.616s step_avg:147.16ms
step:1875/3350 val_loss:3.46527 train_time:275.995s step_avg:147.04ms
step:2000/3350 val_loss:3.44675 train_time:294.383s step_avg:147.10ms
step:2125/3350 val_loss:3.42917 train_time:312.843s step_avg:147.68ms
step:2250/3350 val_loss:3.41090 train_time:331.213s step_avg:146.96ms
step:2375/3350 val_loss:3.39434 train_time:349.605s step_avg:147.13ms
step:2500/3350 val_loss:3.37770 train_time:367.999s step_avg:147.16ms
step:2625/3350 val_loss:3.36043 train_time:386.382s step_avg:147.06ms
step:2750/3350 val_loss:3.34415 train_time:404.774s step_avg:147.13ms
step:2875/3350 val_loss:3.32820 train_time:423.167s step_avg:147.14ms
step:3000/3350 val_loss:3.31220 train_time:441.549s step_avg:147.06ms
step:3025/3350 val_loss:3.30812 train_time:445.234s step_avg:147.42ms
step:3050/3350 val_loss:3.30523 train_time:448.930s step_avg:147.84ms
step:3075/3350 val_loss:3.30234 train_time:452.609s step_avg:147.14ms
step:3100/3350 val_loss:3.29946 train_time:456.283s step_avg:147.00ms
step:3125/3350 val_loss:3.29667 train_time:459.960s step_avg:147.05ms
step:3150/3350 val_loss:3.29356 train_time:463.640s step_avg:147.20ms
step:3175/3350 val_loss:3.29100 train_time:467.315s step_avg:147.00ms
step:3200/3350 val_loss:3.28828 train_time:470.988s step_avg:146.95ms
step:3225/3350 val_loss:3.28602 train_time:474.661s step_avg:146.90ms
step:3250/3350 val_loss:3.28401 train_time:478.359s step_avg:147.93ms
step:3275/3350 val_loss:3.28212 train_time:482.037s step_avg:147.11ms
step:3300/3350 val_loss:3.28053 train_time:485.714s step_avg:147.09ms
step:3325/3350 val_loss:3.27932 train_time:489.388s step_avg:146.97ms
step:3350/3350 val_loss:3.27874 train_time:493.067s step_avg:147.16ms
step:0/3350 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3350 val_loss:4.67080 train_time:18.479s step_avg:147.84ms
step:250/3350 val_loss:4.13310 train_time:36.913s step_avg:147.46ms
step:375/3350 val_loss:3.94243 train_time:55.307s step_avg:147.16ms
step:500/3350 val_loss:3.83458 train_time:73.715s step_avg:147.26ms
step:625/3350 val_loss:3.76732 train_time:92.113s step_avg:147.18ms
step:750/3350 val_loss:3.71666 train_time:110.490s step_avg:147.02ms
step:875/3350 val_loss:3.67697 train_time:128.884s step_avg:147.15ms
step:1000/3350 val_loss:3.64335 train_time:147.276s step_avg:147.13ms
step:1125/3350 val_loss:3.61276 train_time:165.645s step_avg:146.95ms
step:1250/3350 val_loss:3.58110 train_time:184.049s step_avg:147.23ms
step:1375/3350 val_loss:3.55552 train_time:202.435s step_avg:147.09ms
step:1500/3350 val_loss:3.52713 train_time:220.789s step_avg:146.83ms
step:1625/3350 val_loss:3.50744 train_time:239.174s step_avg:147.08ms
step:1750/3350 val_loss:3.48532 train_time:257.558s step_avg:147.07ms
step:1875/3350 val_loss:3.46566 train_time:275.932s step_avg:147.00ms
step:2000/3350 val_loss:3.44614 train_time:294.319s step_avg:147.09ms
step:2125/3350 val_loss:3.42825 train_time:312.694s step_avg:147.00ms
step:2250/3350 val_loss:3.41035 train_time:331.062s step_avg:146.94ms
step:2375/3350 val_loss:3.39379 train_time:349.452s step_avg:147.12ms
step:2500/3350 val_loss:3.37731 train_time:367.851s step_avg:147.19ms
step:2625/3350 val_loss:3.36009 train_time:386.287s step_avg:147.49ms
step:2750/3350 val_loss:3.34384 train_time:404.680s step_avg:147.14ms
step:2875/3350 val_loss:3.32755 train_time:423.065s step_avg:147.08ms
step:3000/3350 val_loss:3.31175 train_time:441.433s step_avg:146.94ms
step:3025/3350 val_loss:3.30792 train_time:445.106s step_avg:146.92ms
step:3050/3350 val_loss:3.30488 train_time:448.799s step_avg:147.71ms
step:3075/3350 val_loss:3.30198 train_time:452.479s step_avg:147.21ms
step:3100/3350 val_loss:3.29900 train_time:456.160s step_avg:147.25ms
step:3125/3350 val_loss:3.29633 train_time:459.834s step_avg:146.94ms
step:3150/3350 val_loss:3.29328 train_time:463.508s step_avg:146.99ms
step:3175/3350 val_loss:3.29071 train_time:467.184s step_avg:147.03ms
step:3200/3350 val_loss:3.28822 train_time:470.861s step_avg:147.08ms
step:3225/3350 val_loss:3.28591 train_time:474.538s step_avg:147.08ms
step:3250/3350 val_loss:3.28385 train_time:478.237s step_avg:147.98ms
step:3275/3350 val_loss:3.28193 train_time:481.913s step_avg:147.01ms
step:3300/3350 val_loss:3.28037 train_time:485.588s step_avg:147.01ms
step:3325/3350 val_loss:3.27920 train_time:489.264s step_avg:147.06ms
step:3350/3350 val_loss:3.27863 train_time:492.942s step_avg:147.12ms
step:0/3350 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3350 val_loss:4.67462 train_time:18.469s step_avg:147.75ms
step:250/3350 val_loss:4.12631 train_time:36.898s step_avg:147.43ms
step:375/3350 val_loss:3.94258 train_time:55.294s step_avg:147.17ms
step:500/3350 val_loss:3.83371 train_time:73.701s step_avg:147.26ms
step:625/3350 val_loss:3.76288 train_time:92.097s step_avg:147.16ms
step:750/3350 val_loss:3.71795 train_time:110.474s step_avg:147.02ms
step:875/3350 val_loss:3.67780 train_time:128.871s step_avg:147.18ms
step:1000/3350 val_loss:3.64128 train_time:147.261s step_avg:147.12ms
step:1125/3350 val_loss:3.61226 train_time:165.628s step_avg:146.94ms
step:1250/3350 val_loss:3.58166 train_time:184.013s step_avg:147.07ms
step:1375/3350 val_loss:3.55541 train_time:202.418s step_avg:147.24ms
step:1500/3350 val_loss:3.52884 train_time:220.783s step_avg:146.92ms
step:1625/3350 val_loss:3.50858 train_time:239.168s step_avg:147.08ms
step:1750/3350 val_loss:3.48568 train_time:257.553s step_avg:147.09ms
step:1875/3350 val_loss:3.46477 train_time:275.929s step_avg:147.01ms
step:2000/3350 val_loss:3.44532 train_time:294.313s step_avg:147.07ms
step:2125/3350 val_loss:3.42759 train_time:312.706s step_avg:147.14ms
step:2250/3350 val_loss:3.40988 train_time:331.079s step_avg:146.99ms
step:2375/3350 val_loss:3.39402 train_time:349.471s step_avg:147.13ms
step:2500/3350 val_loss:3.37694 train_time:367.872s step_avg:147.21ms
step:2625/3350 val_loss:3.36029 train_time:386.233s step_avg:146.89ms
step:2750/3350 val_loss:3.34394 train_time:404.612s step_avg:147.03ms
step:2875/3350 val_loss:3.32781 train_time:423.006s step_avg:147.15ms
step:3000/3350 val_loss:3.31180 train_time:441.376s step_avg:146.96ms
step:3025/3350 val_loss:3.30774 train_time:445.116s step_avg:149.60ms
step:3050/3350 val_loss:3.30496 train_time:448.817s step_avg:148.04ms
step:3075/3350 val_loss:3.30199 train_time:452.499s step_avg:147.26ms
step:3100/3350 val_loss:3.29904 train_time:456.185s step_avg:147.45ms
step:3125/3350 val_loss:3.29634 train_time:459.862s step_avg:147.09ms
step:3150/3350 val_loss:3.29334 train_time:463.532s step_avg:146.81ms
step:3175/3350 val_loss:3.29076 train_time:467.213s step_avg:147.24ms
step:3200/3350 val_loss:3.28822 train_time:470.888s step_avg:146.98ms
step:3225/3350 val_loss:3.28583 train_time:474.565s step_avg:147.08ms
step:3250/3350 val_loss:3.28373 train_time:478.265s step_avg:148.00ms
step:3275/3350 val_loss:3.28185 train_time:481.934s step_avg:146.78ms
step:3300/3350 val_loss:3.28025 train_time:485.609s step_avg:147.01ms
step:3325/3350 val_loss:3.27905 train_time:489.282s step_avg:146.90ms
step:3350/3350 val_loss:3.27847 train_time:492.956s step_avg:146.96ms
step:0/3350 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3350 val_loss:4.67193 train_time:18.453s step_avg:147.62ms
step:250/3350 val_loss:4.13128 train_time:36.881s step_avg:147.42ms
step:375/3350 val_loss:3.93947 train_time:55.274s step_avg:147.14ms
step:500/3350 val_loss:3.83511 train_time:73.680s step_avg:147.25ms
step:625/3350 val_loss:3.76624 train_time:92.080s step_avg:147.20ms
step:750/3350 val_loss:3.71885 train_time:110.467s step_avg:147.10ms
step:875/3350 val_loss:3.67738 train_time:128.862s step_avg:147.16ms
step:1000/3350 val_loss:3.64379 train_time:147.232s step_avg:146.96ms
step:1125/3350 val_loss:3.61372 train_time:165.610s step_avg:147.02ms
step:1250/3350 val_loss:3.58222 train_time:184.000s step_avg:147.12ms
step:1375/3350 val_loss:3.55637 train_time:202.375s step_avg:147.01ms
step:1500/3350 val_loss:3.52758 train_time:220.734s step_avg:146.87ms
step:1625/3350 val_loss:3.50839 train_time:239.114s step_avg:147.04ms
step:1750/3350 val_loss:3.48610 train_time:257.496s step_avg:147.06ms
step:1875/3350 val_loss:3.46542 train_time:275.866s step_avg:146.95ms
step:2000/3350 val_loss:3.44620 train_time:294.255s step_avg:147.11ms
step:2125/3350 val_loss:3.42887 train_time:312.634s step_avg:147.04ms
step:2250/3350 val_loss:3.41066 train_time:330.976s step_avg:146.73ms
step:2375/3350 val_loss:3.39463 train_time:349.336s step_avg:146.88ms
step:2500/3350 val_loss:3.37799 train_time:367.711s step_avg:147.00ms
step:2625/3350 val_loss:3.36040 train_time:386.082s step_avg:146.96ms
step:2750/3350 val_loss:3.34433 train_time:404.469s step_avg:147.10ms
step:2875/3350 val_loss:3.32793 train_time:422.841s step_avg:146.97ms
step:3000/3350 val_loss:3.31202 train_time:441.194s step_avg:146.83ms
step:3025/3350 val_loss:3.30803 train_time:444.864s step_avg:146.80ms
step:3050/3350 val_loss:3.30516 train_time:448.559s step_avg:147.80ms
step:3075/3350 val_loss:3.30232 train_time:452.230s step_avg:146.85ms
step:3100/3350 val_loss:3.29932 train_time:455.900s step_avg:146.79ms
step:3125/3350 val_loss:3.29667 train_time:459.572s step_avg:146.89ms
step:3150/3350 val_loss:3.29362 train_time:463.250s step_avg:147.10ms
step:3175/3350 val_loss:3.29099 train_time:466.927s step_avg:147.10ms
step:3200/3350 val_loss:3.28838 train_time:470.601s step_avg:146.95ms
step:3225/3350 val_loss:3.28608 train_time:474.275s step_avg:146.97ms
step:3250/3350 val_loss:3.28405 train_time:477.965s step_avg:147.59ms
step:3275/3350 val_loss:3.28218 train_time:481.642s step_avg:147.05ms
step:3300/3350 val_loss:3.28060 train_time:485.322s step_avg:147.21ms
step:3325/3350 val_loss:3.27945 train_time:488.990s step_avg:146.75ms
step:3350/3350 val_loss:3.27887 train_time:492.660s step_avg:146.80ms
step:0/3350 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3350 val_loss:4.66758 train_time:18.507s step_avg:148.06ms
step:250/3350 val_loss:4.12682 train_time:36.934s step_avg:147.42ms
step:375/3350 val_loss:3.94177 train_time:55.331s step_avg:147.18ms
step:500/3350 val_loss:3.83481 train_time:73.739s step_avg:147.26ms
step:625/3350 val_loss:3.76504 train_time:92.139s step_avg:147.20ms
step:750/3350 val_loss:3.71569 train_time:110.497s step_avg:146.86ms
step:875/3350 val_loss:3.67503 train_time:128.875s step_avg:147.03ms
step:1000/3350 val_loss:3.64442 train_time:147.248s step_avg:146.98ms
step:1125/3350 val_loss:3.61668 train_time:165.589s step_avg:146.73ms
step:1250/3350 val_loss:3.58082 train_time:183.952s step_avg:146.90ms
step:1375/3350 val_loss:3.55489 train_time:202.333s step_avg:147.05ms
step:1500/3350 val_loss:3.52585 train_time:220.681s step_avg:146.78ms
step:1625/3350 val_loss:3.50733 train_time:239.036s step_avg:146.84ms
step:1750/3350 val_loss:3.48403 train_time:257.404s step_avg:146.94ms
step:1875/3350 val_loss:3.46443 train_time:275.768s step_avg:146.91ms
step:2000/3350 val_loss:3.44490 train_time:294.137s step_avg:146.95ms
step:2125/3350 val_loss:3.42767 train_time:312.530s step_avg:147.14ms
step:2250/3350 val_loss:3.40963 train_time:330.896s step_avg:146.93ms
step:2375/3350 val_loss:3.39339 train_time:349.267s step_avg:146.97ms
step:2500/3350 val_loss:3.37677 train_time:367.645s step_avg:147.02ms
step:2625/3350 val_loss:3.35923 train_time:385.991s step_avg:146.77ms
step:2750/3350 val_loss:3.34339 train_time:404.374s step_avg:147.06ms
step:2875/3350 val_loss:3.32681 train_time:422.751s step_avg:147.02ms
step:3000/3350 val_loss:3.31101 train_time:441.112s step_avg:146.88ms
step:3025/3350 val_loss:3.30715 train_time:444.787s step_avg:147.01ms
step:3050/3350 val_loss:3.30406 train_time:448.482s step_avg:147.81ms
step:3075/3350 val_loss:3.30106 train_time:452.155s step_avg:146.91ms
step:3100/3350 val_loss:3.29827 train_time:455.833s step_avg:147.12ms
step:3125/3350 val_loss:3.29564 train_time:459.504s step_avg:146.83ms
step:3150/3350 val_loss:3.29254 train_time:463.178s step_avg:146.95ms
step:3175/3350 val_loss:3.29001 train_time:466.851s step_avg:146.92ms
step:3200/3350 val_loss:3.28740 train_time:470.528s step_avg:147.11ms
step:3225/3350 val_loss:3.28509 train_time:474.204s step_avg:147.00ms
step:3250/3350 val_loss:3.28304 train_time:477.899s step_avg:147.81ms
step:3275/3350 val_loss:3.28120 train_time:481.573s step_avg:146.97ms
step:3300/3350 val_loss:3.27962 train_time:485.249s step_avg:147.05ms
step:3325/3350 val_loss:3.27845 train_time:488.924s step_avg:146.99ms
step:3350/3350 val_loss:3.27786 train_time:492.601s step_avg:147.10ms
step:0/3350 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3350 val_loss:4.68546 train_time:18.461s step_avg:147.69ms
step:250/3350 val_loss:4.13251 train_time:36.877s step_avg:147.33ms
step:375/3350 val_loss:3.94038 train_time:55.266s step_avg:147.11ms
step:500/3350 val_loss:3.83409 train_time:73.664s step_avg:147.19ms
step:625/3350 val_loss:3.76398 train_time:92.137s step_avg:147.78ms
step:750/3350 val_loss:3.71727 train_time:110.509s step_avg:146.97ms
step:875/3350 val_loss:3.67691 train_time:128.875s step_avg:146.92ms
step:1000/3350 val_loss:3.64247 train_time:147.254s step_avg:147.03ms
step:1125/3350 val_loss:3.61589 train_time:165.605s step_avg:146.81ms
step:1250/3350 val_loss:3.58147 train_time:183.955s step_avg:146.81ms
step:1375/3350 val_loss:3.55447 train_time:202.350s step_avg:147.16ms
step:1500/3350 val_loss:3.52577 train_time:220.700s step_avg:146.80ms
step:1625/3350 val_loss:3.50605 train_time:239.055s step_avg:146.84ms
step:1750/3350 val_loss:3.48425 train_time:257.410s step_avg:146.84ms
step:1875/3350 val_loss:3.46444 train_time:275.753s step_avg:146.74ms
step:2000/3350 val_loss:3.44491 train_time:294.107s step_avg:146.84ms
step:2125/3350 val_loss:3.42712 train_time:312.465s step_avg:146.86ms
step:2250/3350 val_loss:3.40971 train_time:330.818s step_avg:146.82ms
step:2375/3350 val_loss:3.39312 train_time:349.181s step_avg:146.91ms
step:2500/3350 val_loss:3.37694 train_time:367.544s step_avg:146.90ms
step:2625/3350 val_loss:3.35939 train_time:385.892s step_avg:146.78ms
step:2750/3350 val_loss:3.34334 train_time:404.257s step_avg:146.92ms
step:2875/3350 val_loss:3.32704 train_time:422.635s step_avg:147.02ms
step:3000/3350 val_loss:3.31098 train_time:440.979s step_avg:146.75ms
step:3025/3350 val_loss:3.30705 train_time:444.656s step_avg:147.08ms
step:3050/3350 val_loss:3.30415 train_time:448.350s step_avg:147.73ms
step:3075/3350 val_loss:3.30132 train_time:452.019s step_avg:146.78ms
step:3100/3350 val_loss:3.29824 train_time:455.685s step_avg:146.64ms
step:3125/3350 val_loss:3.29563 train_time:459.356s step_avg:146.83ms
step:3150/3350 val_loss:3.29249 train_time:463.028s step_avg:146.89ms
step:3175/3350 val_loss:3.29002 train_time:466.698s step_avg:146.81ms
step:3200/3350 val_loss:3.28750 train_time:470.369s step_avg:146.82ms
step:3225/3350 val_loss:3.28509 train_time:474.043s step_avg:146.95ms
step:3250/3350 val_loss:3.28305 train_time:477.732s step_avg:147.57ms
step:3275/3350 val_loss:3.28117 train_time:481.408s step_avg:147.06ms
step:3300/3350 val_loss:3.27965 train_time:485.091s step_avg:147.30ms
step:3325/3350 val_loss:3.27842 train_time:488.760s step_avg:146.77ms
step:3350/3350 val_loss:3.27784 train_time:492.433s step_avg:146.91ms
step:0/3350 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3350 val_loss:4.67598 train_time:18.444s step_avg:147.55ms
step:250/3350 val_loss:4.12296 train_time:36.880s step_avg:147.49ms
step:375/3350 val_loss:3.93843 train_time:55.275s step_avg:147.15ms
step:500/3350 val_loss:3.83578 train_time:73.678s step_avg:147.23ms
step:625/3350 val_loss:3.76329 train_time:92.071s step_avg:147.15ms
step:750/3350 val_loss:3.71699 train_time:110.449s step_avg:147.02ms
step:875/3350 val_loss:3.67636 train_time:128.811s step_avg:146.90ms
step:1000/3350 val_loss:3.64191 train_time:147.279s step_avg:147.75ms
step:1125/3350 val_loss:3.61517 train_time:165.627s step_avg:146.78ms
step:1250/3350 val_loss:3.58271 train_time:184.000s step_avg:146.99ms
step:1375/3350 val_loss:3.55658 train_time:202.371s step_avg:146.97ms
step:1500/3350 val_loss:3.52768 train_time:220.737s step_avg:146.93ms
step:1625/3350 val_loss:3.50785 train_time:239.117s step_avg:147.04ms
step:1750/3350 val_loss:3.48470 train_time:257.497s step_avg:147.04ms
step:1875/3350 val_loss:3.46568 train_time:275.855s step_avg:146.86ms
step:2000/3350 val_loss:3.44572 train_time:294.220s step_avg:146.92ms
step:2125/3350 val_loss:3.42809 train_time:312.575s step_avg:146.84ms
step:2250/3350 val_loss:3.41050 train_time:330.926s step_avg:146.81ms
step:2375/3350 val_loss:3.39429 train_time:349.284s step_avg:146.86ms
step:2500/3350 val_loss:3.37768 train_time:367.646s step_avg:146.90ms
step:2625/3350 val_loss:3.36009 train_time:385.997s step_avg:146.80ms
step:2750/3350 val_loss:3.34365 train_time:404.386s step_avg:147.12ms
step:2875/3350 val_loss:3.32753 train_time:422.753s step_avg:146.93ms
step:3000/3350 val_loss:3.31161 train_time:441.111s step_avg:146.86ms
step:3025/3350 val_loss:3.30785 train_time:444.788s step_avg:147.11ms
step:3050/3350 val_loss:3.30482 train_time:448.475s step_avg:147.45ms
step:3075/3350 val_loss:3.30183 train_time:452.148s step_avg:146.92ms
step:3100/3350 val_loss:3.29891 train_time:455.825s step_avg:147.08ms
step:3125/3350 val_loss:3.29637 train_time:459.502s step_avg:147.07ms
step:3150/3350 val_loss:3.29324 train_time:463.178s step_avg:147.04ms
step:3175/3350 val_loss:3.29067 train_time:466.852s step_avg:146.97ms
step:3200/3350 val_loss:3.28810 train_time:470.529s step_avg:147.09ms
step:3225/3350 val_loss:3.28588 train_time:474.209s step_avg:147.18ms
step:3250/3350 val_loss:3.28384 train_time:477.907s step_avg:147.94ms
step:3275/3350 val_loss:3.28205 train_time:481.582s step_avg:146.98ms
step:3300/3350 val_loss:3.28044 train_time:485.256s step_avg:146.98ms
step:3325/3350 val_loss:3.27922 train_time:488.933s step_avg:147.06ms
step:3350/3350 val_loss:3.27864 train_time:492.608s step_avg:147.00ms
step:0/3350 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3350 val_loss:4.66927 train_time:18.457s step_avg:147.65ms
step:250/3350 val_loss:4.12390 train_time:36.887s step_avg:147.44ms
step:375/3350 val_loss:3.93775 train_time:55.276s step_avg:147.11ms
step:500/3350 val_loss:3.83220 train_time:73.671s step_avg:147.16ms
step:625/3350 val_loss:3.76166 train_time:92.061s step_avg:147.12ms
step:750/3350 val_loss:3.71500 train_time:110.419s step_avg:146.87ms
step:875/3350 val_loss:3.67614 train_time:128.808s step_avg:147.11ms
step:1000/3350 val_loss:3.64244 train_time:147.184s step_avg:147.01ms
step:1125/3350 val_loss:3.61348 train_time:165.528s step_avg:146.75ms
step:1250/3350 val_loss:3.58038 train_time:183.897s step_avg:146.95ms
step:1375/3350 val_loss:3.55448 train_time:202.265s step_avg:146.94ms
step:1500/3350 val_loss:3.52548 train_time:220.709s step_avg:147.56ms
step:1625/3350 val_loss:3.50807 train_time:239.075s step_avg:146.93ms
step:1750/3350 val_loss:3.48388 train_time:257.439s step_avg:146.91ms
step:1875/3350 val_loss:3.46318 train_time:275.774s step_avg:146.68ms
step:2000/3350 val_loss:3.44421 train_time:294.131s step_avg:146.86ms
step:2125/3350 val_loss:3.42626 train_time:312.482s step_avg:146.81ms
step:2250/3350 val_loss:3.40885 train_time:330.832s step_avg:146.80ms
step:2375/3350 val_loss:3.39237 train_time:349.196s step_avg:146.92ms
step:2500/3350 val_loss:3.37545 train_time:367.552s step_avg:146.85ms
step:2625/3350 val_loss:3.35840 train_time:385.915s step_avg:146.91ms
step:2750/3350 val_loss:3.34210 train_time:404.298s step_avg:147.06ms
step:2875/3350 val_loss:3.32626 train_time:422.657s step_avg:146.87ms
step:3000/3350 val_loss:3.31030 train_time:440.999s step_avg:146.74ms
step:3025/3350 val_loss:3.30614 train_time:444.676s step_avg:147.06ms
step:3050/3350 val_loss:3.30323 train_time:448.370s step_avg:147.75ms
step:3075/3350 val_loss:3.30040 train_time:452.037s step_avg:146.70ms
step:3100/3350 val_loss:3.29750 train_time:455.709s step_avg:146.87ms
step:3125/3350 val_loss:3.29469 train_time:459.386s step_avg:147.07ms
step:3150/3350 val_loss:3.29163 train_time:463.062s step_avg:147.03ms
step:3175/3350 val_loss:3.28901 train_time:466.739s step_avg:147.11ms
step:3200/3350 val_loss:3.28647 train_time:470.417s step_avg:147.10ms
step:3225/3350 val_loss:3.28412 train_time:474.087s step_avg:146.79ms
step:3250/3350 val_loss:3.28212 train_time:477.782s step_avg:147.80ms
step:3275/3350 val_loss:3.28031 train_time:481.455s step_avg:146.93ms
step:3300/3350 val_loss:3.27873 train_time:485.129s step_avg:146.98ms
step:3325/3350 val_loss:3.27754 train_time:488.807s step_avg:147.12ms
step:3350/3350 val_loss:3.27695 train_time:492.486s step_avg:147.13ms
step:0/3350 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3350 val_loss:4.67464 train_time:18.446s step_avg:147.57ms
step:250/3350 val_loss:4.12866 train_time:36.869s step_avg:147.38ms
step:375/3350 val_loss:3.94074 train_time:55.245s step_avg:147.01ms
step:500/3350 val_loss:3.83472 train_time:73.648s step_avg:147.23ms
step:625/3350 val_loss:3.76582 train_time:92.046s step_avg:147.18ms
step:750/3350 val_loss:3.71923 train_time:110.415s step_avg:146.95ms
step:875/3350 val_loss:3.67800 train_time:128.789s step_avg:147.00ms
step:1000/3350 val_loss:3.64481 train_time:147.172s step_avg:147.06ms
step:1125/3350 val_loss:3.61748 train_time:165.508s step_avg:146.69ms
step:1250/3350 val_loss:3.58259 train_time:183.873s step_avg:146.92ms
step:1375/3350 val_loss:3.55662 train_time:202.258s step_avg:147.08ms
step:1500/3350 val_loss:3.52921 train_time:220.597s step_avg:146.72ms
step:1625/3350 val_loss:3.50980 train_time:238.969s step_avg:146.98ms
step:1750/3350 val_loss:3.48684 train_time:257.352s step_avg:147.06ms
step:1875/3350 val_loss:3.46530 train_time:275.687s step_avg:146.68ms
step:2000/3350 val_loss:3.44636 train_time:294.126s step_avg:147.51ms
step:2125/3350 val_loss:3.42902 train_time:312.504s step_avg:147.03ms
step:2250/3350 val_loss:3.41062 train_time:330.853s step_avg:146.79ms
step:2375/3350 val_loss:3.39423 train_time:349.231s step_avg:147.03ms
step:2500/3350 val_loss:3.37809 train_time:367.593s step_avg:146.90ms
step:2625/3350 val_loss:3.36091 train_time:385.947s step_avg:146.83ms
step:2750/3350 val_loss:3.34476 train_time:404.303s step_avg:146.84ms
step:2875/3350 val_loss:3.32835 train_time:422.659s step_avg:146.85ms
step:3000/3350 val_loss:3.31208 train_time:441.009s step_avg:146.80ms
step:3025/3350 val_loss:3.30809 train_time:444.680s step_avg:146.82ms
step:3050/3350 val_loss:3.30513 train_time:448.366s step_avg:147.44ms
step:3075/3350 val_loss:3.30215 train_time:452.036s step_avg:146.81ms
step:3100/3350 val_loss:3.29944 train_time:455.711s step_avg:147.01ms
step:3125/3350 val_loss:3.29663 train_time:459.379s step_avg:146.73ms
step:3150/3350 val_loss:3.29359 train_time:463.056s step_avg:147.06ms
step:3175/3350 val_loss:3.29094 train_time:466.732s step_avg:147.04ms
step:3200/3350 val_loss:3.28839 train_time:470.402s step_avg:146.80ms
step:3225/3350 val_loss:3.28597 train_time:474.074s step_avg:146.89ms
step:3250/3350 val_loss:3.28398 train_time:477.767s step_avg:147.71ms
step:3275/3350 val_loss:3.28214 train_time:481.439s step_avg:146.89ms
step:3300/3350 val_loss:3.28057 train_time:485.112s step_avg:146.92ms
step:3325/3350 val_loss:3.27938 train_time:488.789s step_avg:147.08ms
step:3350/3350 val_loss:3.27880 train_time:492.463s step_avg:146.93ms
step:0/3350 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3350 val_loss:4.67499 train_time:18.445s step_avg:147.56ms
step:250/3350 val_loss:4.12871 train_time:36.863s step_avg:147.34ms
step:375/3350 val_loss:3.94752 train_time:55.240s step_avg:147.02ms
step:500/3350 val_loss:3.83678 train_time:73.634s step_avg:147.15ms
step:625/3350 val_loss:3.76675 train_time:92.020s step_avg:147.09ms
step:750/3350 val_loss:3.71984 train_time:110.380s step_avg:146.88ms
step:875/3350 val_loss:3.67909 train_time:128.776s step_avg:147.17ms
step:1000/3350 val_loss:3.64330 train_time:147.151s step_avg:146.99ms
step:1125/3350 val_loss:3.61639 train_time:165.509s step_avg:146.87ms
step:1250/3350 val_loss:3.58256 train_time:183.870s step_avg:146.88ms
step:1375/3350 val_loss:3.55714 train_time:202.238s step_avg:146.95ms
step:1500/3350 val_loss:3.53241 train_time:220.588s step_avg:146.80ms
step:1625/3350 val_loss:3.50996 train_time:238.966s step_avg:147.02ms
step:1750/3350 val_loss:3.48835 train_time:257.328s step_avg:146.89ms
step:1875/3350 val_loss:3.46831 train_time:275.679s step_avg:146.81ms
step:2000/3350 val_loss:3.44857 train_time:294.045s step_avg:146.93ms
step:2125/3350 val_loss:3.43070 train_time:312.429s step_avg:147.07ms
step:2250/3350 val_loss:3.41269 train_time:330.790s step_avg:146.89ms
step:2375/3350 val_loss:3.39645 train_time:349.152s step_avg:146.90ms
step:2500/3350 val_loss:3.37998 train_time:367.569s step_avg:147.33ms
step:2625/3350 val_loss:3.36234 train_time:385.920s step_avg:146.80ms
step:2750/3350 val_loss:3.34626 train_time:404.272s step_avg:146.82ms
step:2875/3350 val_loss:3.33023 train_time:422.624s step_avg:146.82ms
step:3000/3350 val_loss:3.31422 train_time:440.964s step_avg:146.71ms
step:3025/3350 val_loss:3.31026 train_time:444.636s step_avg:146.89ms
step:3050/3350 val_loss:3.30725 train_time:448.333s step_avg:147.87ms
step:3075/3350 val_loss:3.30447 train_time:452.005s step_avg:146.88ms
step:3100/3350 val_loss:3.30152 train_time:455.682s step_avg:147.07ms
step:3125/3350 val_loss:3.29882 train_time:459.357s step_avg:147.03ms
step:3150/3350 val_loss:3.29580 train_time:463.032s step_avg:146.98ms
step:3175/3350 val_loss:3.29317 train_time:466.700s step_avg:146.75ms
step:3200/3350 val_loss:3.29061 train_time:470.377s step_avg:147.08ms
step:3225/3350 val_loss:3.28823 train_time:474.050s step_avg:146.91ms
step:3250/3350 val_loss:3.28619 train_time:477.743s step_avg:147.69ms
step:3275/3350 val_loss:3.28432 train_time:481.418s step_avg:147.02ms
step:3300/3350 val_loss:3.28276 train_time:485.090s step_avg:146.88ms
step:3325/3350 val_loss:3.28157 train_time:488.763s step_avg:146.92ms
step:3350/3350 val_loss:3.28099 train_time:492.441s step_avg:147.13ms
step:0/3350 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3350 val_loss:4.68087 train_time:18.451s step_avg:147.61ms
step:250/3350 val_loss:4.13291 train_time:36.861s step_avg:147.28ms
step:375/3350 val_loss:3.94042 train_time:55.254s step_avg:147.14ms
step:500/3350 val_loss:3.83622 train_time:73.644s step_avg:147.12ms
step:625/3350 val_loss:3.76609 train_time:92.028s step_avg:147.07ms
step:750/3350 val_loss:3.71880 train_time:110.398s step_avg:146.96ms
step:875/3350 val_loss:3.67727 train_time:128.778s step_avg:147.05ms
step:1000/3350 val_loss:3.64208 train_time:147.157s step_avg:147.03ms
step:1125/3350 val_loss:3.61510 train_time:165.522s step_avg:146.93ms
step:1250/3350 val_loss:3.58175 train_time:183.896s step_avg:146.99ms
step:1375/3350 val_loss:3.55507 train_time:202.268s step_avg:146.97ms
step:1500/3350 val_loss:3.52571 train_time:220.613s step_avg:146.76ms
step:1625/3350 val_loss:3.50672 train_time:238.981s step_avg:146.95ms
step:1750/3350 val_loss:3.48518 train_time:257.357s step_avg:147.00ms
step:1875/3350 val_loss:3.46410 train_time:275.708s step_avg:146.81ms
step:2000/3350 val_loss:3.44514 train_time:294.079s step_avg:146.97ms
step:2125/3350 val_loss:3.42774 train_time:312.445s step_avg:146.92ms
step:2250/3350 val_loss:3.40979 train_time:330.800s step_avg:146.85ms
step:2375/3350 val_loss:3.39302 train_time:349.172s step_avg:146.98ms
step:2500/3350 val_loss:3.37684 train_time:367.548s step_avg:147.00ms
step:2625/3350 val_loss:3.35964 train_time:385.896s step_avg:146.78ms
step:2750/3350 val_loss:3.34361 train_time:404.258s step_avg:146.90ms
step:2875/3350 val_loss:3.32752 train_time:422.700s step_avg:147.54ms
step:3000/3350 val_loss:3.31168 train_time:441.051s step_avg:146.80ms
step:3025/3350 val_loss:3.30767 train_time:444.717s step_avg:146.63ms
step:3050/3350 val_loss:3.30477 train_time:448.401s step_avg:147.38ms
step:3075/3350 val_loss:3.30185 train_time:452.077s step_avg:147.04ms
step:3100/3350 val_loss:3.29876 train_time:455.744s step_avg:146.69ms
step:3125/3350 val_loss:3.29609 train_time:459.412s step_avg:146.69ms
step:3150/3350 val_loss:3.29313 train_time:463.083s step_avg:146.84ms
step:3175/3350 val_loss:3.29051 train_time:466.759s step_avg:147.04ms
step:3200/3350 val_loss:3.28786 train_time:470.434s step_avg:147.02ms
step:3225/3350 val_loss:3.28563 train_time:474.104s step_avg:146.81ms
step:3250/3350 val_loss:3.28360 train_time:477.794s step_avg:147.57ms
step:3275/3350 val_loss:3.28174 train_time:481.461s step_avg:146.70ms
step:3300/3350 val_loss:3.28018 train_time:485.134s step_avg:146.90ms
step:3325/3350 val_loss:3.27898 train_time:488.809s step_avg:147.00ms
step:3350/3350 val_loss:3.27839 train_time:492.485s step_avg:147.03ms