"""
train_gpt_simple.py

This file descends from the [NanoGPT speedrun](https://github.com/KellerJordan/modded-nanogpt).
It was prepared as a simplified version of the speedrun for use in neural net optimization research.
"""

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
from pathlib import Path

import torch
from torch import Tensor, nn
from torch.optim import AdamW
import torch.nn.functional as F
import torch.distributed as dist


########################################
#              Dataloader              #
########################################

def _load_data_shard(file: Path):
    header = torch.from_file(str(file), False, 256, dtype=torch.int32) # header is 256 int32
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    num_tokens = int(header[2]) # number of tokens (claimed)
    with file.open("rb", buffering=0) as f:
        tokens = torch.empty(num_tokens, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy()) # avoid bytes->array copy
        assert nbytes == 2 * num_tokens, "number of tokens read does not match header"
    return tokens

def distributed_data_generator(filename_pattern: str, batch_size: int, seq_len=1024):
    world_size = dist.get_world_size()
    rank = dist.get_rank()
    files = sorted(Path.cwd().glob(filename_pattern))
    assert batch_size % world_size == 0
    local_batch_size = batch_size // world_size
    file_iter = iter(files)
    tokens, pos = _load_data_shard(next(file_iter)), 0
    while True:
        if pos + batch_size + 1 >= len(tokens):
            tokens, pos = _load_data_shard(next(file_iter)), 0
        buf = tokens[pos + rank * local_batch_size:][:local_batch_size + 1]
        inputs = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True)
        targets = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True)
        pos += batch_size
        yield inputs.view(-1, seq_len), targets.view(-1, seq_len)


########################################
#             Architecture             #
########################################

class RMSNorm(nn.Module):
    def __init__(self, dim):
        super().__init__()
        self.gains = nn.Parameter(torch.ones(dim))

    def forward(self, x):
        return F.rms_norm(x, (x.size(-1),), weight=self.gains.type_as(x))

class Linear(nn.Linear):
    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=True)

    def forward(self, x):
        return F.linear(x, self.weight.type_as(x), self.bias.type_as(x))

class Rotary(nn.Module):
    def __init__(self, dim: int):
        super().__init__()
        # half-truncate RoPE (w/ base freq tuning)
        angular_freq = (1 / 1024) ** torch.linspace(0, 1, steps=dim//4, dtype=torch.float32)
        self.register_buffer("angular_freq", torch.cat([angular_freq, angular_freq.new_zeros(dim//4)]))

    def forward(self, x_BTHD: Tensor):
        pos = torch.arange(x_BTHD.size(1), dtype=torch.float32, device=x_BTHD.device)
        theta = torch.outer(pos, self.angular_freq)[None, :, None, :]
        cos, sin = theta.cos(), theta.sin()
        x1, x2 = x_BTHD.to(dtype=torch.float32).chunk(2, dim=-1)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x_BTHD)

class CausalSelfAttention(nn.Module):
    def __init__(self, dim: int, head_dim=128):
        super().__init__()
        self.num_heads = dim // head_dim
        self.head_dim = head_dim
        hdim = self.num_heads * self.head_dim
        self.q = Linear(dim, hdim)
        self.k = Linear(dim, hdim)
        self.v = Linear(dim, hdim)
        self.proj = Linear(hdim, dim)
        self.rotary = Rotary(head_dim)

    def forward(self, x: Tensor):
        B, T = x.size(0), x.size(1)
        q = self.q(x).view(B, T, self.num_heads, self.head_dim)
        k = self.k(x).view(B, T, self.num_heads, self.head_dim)
        v = self.v(x).view(B, T, self.num_heads, self.head_dim)
        q, k = F.rms_norm(q, (q.size(-1),)), F.rms_norm(k, (k.size(-1),))
        q, k = self.rotary(q), self.rotary(k)
        y = F.scaled_dot_product_attention(q.transpose(1, 2), k.transpose(1, 2),
                                           v.transpose(1, 2), scale=0.12, is_causal=True).transpose(1, 2)
        y = y.contiguous().view(B, T, self.num_heads * self.head_dim)
        y = self.proj(y)
        return y

class MLP(nn.Module):
    def __init__(self, dim: int):
        super().__init__()
        hdim = 4 * dim
        self.fc = Linear(dim, hdim)
        self.proj = Linear(hdim, dim)

    def forward(self, x: Tensor):
        x = self.fc(x)
        x = x.relu().square()
        x = self.proj(x)
        return x

class Block(nn.Module):
    def __init__(self, dim: int):
        super().__init__()
        self.attn = CausalSelfAttention(dim)
        self.mlp = MLP(dim)
        self.norm1 = RMSNorm(dim)
        self.norm2 = RMSNorm(dim)

    def forward(self, x: Tensor):
        x = x + self.attn(self.norm1(x))
        x = x + self.mlp(self.norm2(x))
        return x

class GPT(nn.Module):
    def __init__(self, vocab_size: int, num_layers: int, model_dim: int):
        super().__init__()
        self.embed = nn.Embedding(vocab_size, model_dim).bfloat16()
        self.blocks = nn.ModuleList([Block(model_dim) for _ in range(num_layers)])
        self.proj = Linear(model_dim, vocab_size)
        self.norm1 = RMSNorm(model_dim)
        self.norm2 = RMSNorm(model_dim)

    def forward(self, inputs: Tensor, targets: Tensor):
        x = self.norm1(self.embed(inputs))
        for block in self.blocks:
            x = block(x)
        logits = self.proj(self.norm2(x)).float()
        logits = 15 * logits * (logits.square() + 15**2).rsqrt()
        return F.cross_entropy(logits.view(targets.numel(), -1), targets.view(-1), reduction="sum")


########################################
#              Optimizer               #
########################################

def zeropower_via_newtonschulz5(G: Tensor) -> Tensor:
    assert G.ndim >= 2
    X = G.bfloat16()
    if G.size(-2) > G.size(-1):
        X = X.mT

    # Ensure spectral norm is at most 1
    X = X / (X.norm(dim=(-2, -1), keepdim=True) + 1e-7)
    # Perform the NS iterations, not optimizing for wallclock speed
    a, b, c = 2, -1.5, 0.5
    for _ in range(12):
        A = X @ X.mT
        B = b * A + c * A @ A
        X = a * X + B @ X

    if G.size(-2) > G.size(-1):
        X = X.mT
    return X

@torch.compile
def muon_update(grad, momentum, mu=0.95, nesterov=True):
    momentum.lerp_(grad, 1 - mu)
    update = grad.lerp_(momentum, mu) if nesterov else momentum
    update = zeropower_via_newtonschulz5(update)
    update *= max(1, grad.size(-2) / grad.size(-1))**0.5
    return update

class Muon(torch.optim.Optimizer):
    def __init__(self, params, lr=0.02, weight_decay=0, mu=0.95):
        assert isinstance(params, list) and len(params) >= 1 and isinstance(params[0], torch.nn.Parameter)
        params = sorted(params, key=lambda x: x.size(), reverse=True)
        defaults = dict(lr=lr, weight_decay=weight_decay, mu=mu)
        super().__init__(params, defaults)

    @torch.no_grad()
    def step(self):
        world_size = dist.get_world_size()
        rank = dist.get_rank()
        for group in self.param_groups:
            params = group["params"]
            params_pad = params + [torch.empty_like(params[-1])] * (world_size - len(params) % world_size)
            for base_i in range(0, len(params), world_size):
                if base_i + rank < len(params):
                    p = params[base_i + rank]
                    state = self.state[p]
                    if len(state) == 0:
                        state["momentum"] = torch.zeros_like(p)
                    update = muon_update(p.grad, state["momentum"], mu=group["mu"])
                    p.mul_(1 - group["lr"] * group["weight_decay"])
                    p.add_(update, alpha=-group["lr"])
                dist.all_gather(params_pad[base_i:base_i + world_size], params_pad[base_i + rank])


########################################
#                Setup                 #
########################################

# torchrun sets these env variables
device = torch.device("cuda", int(os.environ["LOCAL_RANK"]))
torch.cuda.set_device(device)
dist.init_process_group(backend="nccl", device_id=device)
dist.barrier()
# this code can be run equivalently with 1, 2, 4, or 8 gpus.
assert 8 % dist.get_world_size() == 0

# logging setup
if dist.get_rank() == 0:
    os.makedirs("logs", exist_ok=True)
    logfile = f"logs/{uuid.uuid4()}.txt"
    print(logfile)
def print0(s, console=False, log=True):
    if dist.get_rank() == 0:
        if console:
            print(s)
        if log:
            with open(logfile, "a") as f:
                print(s, file=f)

# we begin by logging this file itself
print0(code)
print0("="*100)
print0(f"Running PyTorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}"
       + f" on {torch.cuda.get_device_name(device)} with world_size {dist.get_world_size()}")
print0("="*100)

val_tokens = 20 * 524288
batch_size = 8 * 64 * 1024
mbs = 64
val_inputs, val_targets = next(distributed_data_generator("data/fineweb10B/fineweb_val_*.bin", val_tokens))

model = GPT(vocab_size=50304, num_layers=12, model_dim=768).cuda()
model.compile(dynamic=False)


num_trials = int(sys.argv[-1]) if len(sys.argv) > 1 else 1

for _ in range(num_trials):


    ########################################
    #       Init & Optim Hyperparams       #
    ########################################

    # we want to minimize this while still reaching 3.28 val loss
    train_steps = 3375

    # initialize model parameters
    for name, p in model.named_parameters():
        if name.endswith("weight"):
            if "proj" in name:
                p.data.zero_()
            elif "embed" in name:
                p.data.normal_()  # default torch init
            else:
                p.data.normal_(std=0.33**0.5 / p.size(-1)**0.5)  # default torch init
        elif name.endswith("bias"):
            p.data.zero_()
        elif name.endswith("gains"):
            p.data.normal_(mean=1, std=0)
        else:
            raise Exception(f"Uninitialized parameter: {name}")

    # create the optimizer(s)
    optimizer1 = AdamW([dict(params=[model.embed.weight], lr=0.3),
                        dict(params=[model.proj.weight], lr=1/320),
                        dict(params=[p for p in model.parameters() if p.ndim < 2], lr=0.01)],
                       betas=(0.8, 0.95), eps=1e-10, weight_decay=0, fused=True)
    optimizer2 = Muon([p for p in model.blocks.parameters() if p.ndim >= 2],
                      lr=0.025, weight_decay=0.025)
    optimizers = [optimizer1, optimizer2]
    assert set(p for opt in optimizers for group in opt.param_groups
               for p in group["params"]) == set(model.parameters())
    for opt in optimizers:
        for group in opt.param_groups:
            group["initial_lr"] = group["lr"]

    # learning rate schedule: stable then decay
    def set_hparams(step, cooldown_frac=0.7):
        progress = step / train_steps
        assert 0 <= progress < 1
        if progress < 1 - cooldown_frac:
            eta = 1.0
        else:
            eta = (1 - progress) / cooldown_frac
        for opt in optimizers:
            for group in opt.param_groups:
                group["lr"] = group["initial_lr"] * eta


    ########################################
    #        Training and Validation       #
    ########################################

    train_loader = distributed_data_generator("data/fineweb10B/fineweb_train_*.bin", batch_size)
    for p in model.parameters():
        dist.broadcast(p.detach(), 0)
    # start the clock
    training_time = 0
    last_val_step = 0
    dist.barrier()
    t0 = time.perf_counter()
    for step in range(train_steps + 1):

        # --------------- VALIDATION SECTION -----------------
        if step == train_steps or step % 125 == 0:
            # stop the clock
            dist.barrier()
            time_since_last_val = time.perf_counter() - t0
            step_avg = time_since_last_val / (step - last_val_step) if step > 0 else float("nan")
            last_val_step = step
            training_time += time_since_last_val
            model.eval()
            val_loss = 0
            with torch.no_grad():
                assert len(val_inputs) % mbs == 0
                for i in range(len(val_inputs) // mbs):
                    val_loss += model(val_inputs[i*mbs:(i+1)*mbs], val_targets[i*mbs:(i+1)*mbs])
            dist.all_reduce(val_loss, op=dist.ReduceOp.SUM)
            val_loss /= val_tokens
            print0(f"step:{step}/{train_steps} val_loss:{val_loss:.5f} train_time:{training_time:.3f}s"
                   + f" step_avg:{1000*step_avg:.2f}ms", console=True)
            model.train()
            # start the clock again
            dist.barrier()
            t0 = time.perf_counter()

        if step == train_steps:
            break

        # --------------- TRAINING SECTION -----------------
        inputs, targets = next(train_loader)
        # accumulate across microbatches in case we are running with fewer than 8 gpus
        assert len(inputs) % mbs == 0
        for i in range(len(inputs) // mbs):
            model(inputs[i*mbs:(i+1)*mbs], targets[i*mbs:(i+1)*mbs]).backward()
        for name, p in model.named_parameters():
            assert p.grad is not None, name
            dist.all_reduce(p.grad, op=dist.ReduceOp.SUM)
        # set optimization hyperparameters and take a step
        set_hparams(step)
        for opt in optimizers:
            opt.step()
        model.zero_grad(set_to_none=True)
        approx_training_time = training_time + (time.perf_counter() - t0)
        print0(f"step:{step+1}/{train_steps} train_time:{approx_training_time:.3f}s"
               + f" step_avg:{1000*approx_training_time/(step + 1):.2f}ms", console=True, log=False)

dist.destroy_process_group()

====================================================================================================
Running PyTorch 2.11.0+cu130 compiled for CUDA 13.0 on NVIDIA H100 80GB HBM3 with world_size 8
====================================================================================================
step:0/3375 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3375 val_loss:4.62916 train_time:60.070s step_avg:480.56ms
step:250/3375 val_loss:4.11018 train_time:78.517s step_avg:147.58ms
step:375/3375 val_loss:3.92759 train_time:96.909s step_avg:147.13ms
step:500/3375 val_loss:3.82279 train_time:115.362s step_avg:147.63ms
step:625/3375 val_loss:3.75233 train_time:133.841s step_avg:147.83ms
step:750/3375 val_loss:3.70673 train_time:152.291s step_avg:147.60ms
step:875/3375 val_loss:3.66475 train_time:170.729s step_avg:147.51ms
step:1000/3375 val_loss:3.63309 train_time:189.142s step_avg:147.30ms
step:1125/3375 val_loss:3.60152 train_time:207.490s step_avg:146.79ms
step:1250/3375 val_loss:3.56875 train_time:225.834s step_avg:146.76ms
step:1375/3375 val_loss:3.54333 train_time:244.156s step_avg:146.58ms
step:1500/3375 val_loss:3.51481 train_time:262.453s step_avg:146.37ms
step:1625/3375 val_loss:3.49475 train_time:280.738s step_avg:146.28ms
step:1750/3375 val_loss:3.47261 train_time:299.016s step_avg:146.23ms
step:1875/3375 val_loss:3.45332 train_time:317.281s step_avg:146.12ms
step:2000/3375 val_loss:3.43444 train_time:335.552s step_avg:146.17ms
step:2125/3375 val_loss:3.41702 train_time:353.826s step_avg:146.19ms
step:2250/3375 val_loss:3.40027 train_time:372.068s step_avg:145.94ms
step:2375/3375 val_loss:3.38436 train_time:390.340s step_avg:146.18ms
step:2500/3375 val_loss:3.36927 train_time:408.628s step_avg:146.30ms
step:2625/3375 val_loss:3.35354 train_time:426.922s step_avg:146.35ms
step:2750/3375 val_loss:3.33890 train_time:445.232s step_avg:146.48ms
step:2875/3375 val_loss:3.32427 train_time:463.535s step_avg:146.43ms
step:3000/3375 val_loss:3.31026 train_time:481.823s step_avg:146.30ms
step:3125/3375 val_loss:3.29669 train_time:500.115s step_avg:146.34ms
step:3250/3375 val_loss:3.28526 train_time:518.419s step_avg:146.43ms
step:3375/3375 val_loss:3.27924 train_time:536.697s step_avg:146.23ms
step:0/3375 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3375 val_loss:4.63860 train_time:18.389s step_avg:147.11ms
step:250/3375 val_loss:4.10776 train_time:36.731s step_avg:146.74ms
step:375/3375 val_loss:3.92785 train_time:55.046s step_avg:146.52ms
step:500/3375 val_loss:3.82474 train_time:73.382s step_avg:146.69ms
step:625/3375 val_loss:3.75318 train_time:91.761s step_avg:147.04ms
step:750/3375 val_loss:3.70570 train_time:110.043s step_avg:146.25ms
step:875/3375 val_loss:3.66508 train_time:128.344s step_avg:146.41ms
step:1000/3375 val_loss:3.62898 train_time:146.637s step_avg:146.34ms
step:1125/3375 val_loss:3.59962 train_time:164.902s step_avg:146.12ms
step:1250/3375 val_loss:3.56853 train_time:183.191s step_avg:146.31ms
step:1375/3375 val_loss:3.54213 train_time:201.459s step_avg:146.15ms
step:1500/3375 val_loss:3.51420 train_time:219.742s step_avg:146.27ms
step:1625/3375 val_loss:3.49427 train_time:238.033s step_avg:146.33ms
step:1750/3375 val_loss:3.47286 train_time:256.340s step_avg:146.46ms
step:1875/3375 val_loss:3.45358 train_time:274.612s step_avg:146.18ms
step:2000/3375 val_loss:3.43491 train_time:292.918s step_avg:146.45ms
step:2125/3375 val_loss:3.41763 train_time:311.209s step_avg:146.33ms
step:2250/3375 val_loss:3.40060 train_time:329.491s step_avg:146.25ms
step:2375/3375 val_loss:3.38514 train_time:347.767s step_avg:146.21ms
step:2500/3375 val_loss:3.36989 train_time:366.008s step_avg:145.92ms
step:2625/3375 val_loss:3.35344 train_time:384.236s step_avg:145.82ms
step:2750/3375 val_loss:3.33906 train_time:402.499s step_avg:146.11ms
step:2875/3375 val_loss:3.32434 train_time:420.753s step_avg:146.03ms
step:3000/3375 val_loss:3.31051 train_time:439.011s step_avg:146.06ms
step:3125/3375 val_loss:3.29675 train_time:457.265s step_avg:146.03ms
step:3250/3375 val_loss:3.28548 train_time:475.532s step_avg:146.14ms
step:3375/3375 val_loss:3.27941 train_time:493.801s step_avg:146.15ms
step:0/3375 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3375 val_loss:4.65048 train_time:18.351s step_avg:146.81ms
step:250/3375 val_loss:4.11150 train_time:36.676s step_avg:146.60ms
step:375/3375 val_loss:3.92688 train_time:54.974s step_avg:146.39ms
step:500/3375 val_loss:3.82164 train_time:73.244s step_avg:146.16ms
step:625/3375 val_loss:3.75070 train_time:91.526s step_avg:146.26ms
step:750/3375 val_loss:3.70207 train_time:109.812s step_avg:146.29ms
step:875/3375 val_loss:3.66216 train_time:128.102s step_avg:146.31ms
step:1000/3375 val_loss:3.62771 train_time:146.397s step_avg:146.36ms
step:1125/3375 val_loss:3.59833 train_time:164.733s step_avg:146.69ms
step:1250/3375 val_loss:3.56676 train_time:183.037s step_avg:146.43ms
step:1375/3375 val_loss:3.53929 train_time:201.333s step_avg:146.37ms
step:1500/3375 val_loss:3.51246 train_time:219.606s step_avg:146.18ms
step:1625/3375 val_loss:3.49167 train_time:237.891s step_avg:146.28ms
step:1750/3375 val_loss:3.47041 train_time:256.175s step_avg:146.27ms
step:1875/3375 val_loss:3.45134 train_time:274.444s step_avg:146.15ms
step:2000/3375 val_loss:3.43183 train_time:292.736s step_avg:146.33ms
step:2125/3375 val_loss:3.41487 train_time:311.036s step_avg:146.40ms
step:2250/3375 val_loss:3.39845 train_time:329.312s step_avg:146.20ms
step:2375/3375 val_loss:3.38296 train_time:347.602s step_avg:146.32ms
step:2500/3375 val_loss:3.36714 train_time:365.901s step_avg:146.39ms
step:2625/3375 val_loss:3.35132 train_time:384.172s step_avg:146.17ms
step:2750/3375 val_loss:3.33687 train_time:402.468s step_avg:146.36ms
step:2875/3375 val_loss:3.32227 train_time:420.761s step_avg:146.35ms
step:3000/3375 val_loss:3.30812 train_time:439.033s step_avg:146.18ms
step:3125/3375 val_loss:3.29450 train_time:457.320s step_avg:146.29ms
step:3250/3375 val_loss:3.28295 train_time:475.607s step_avg:146.30ms
step:3375/3375 val_loss:3.27694 train_time:493.880s step_avg:146.19ms
step:0/3375 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3375 val_loss:4.63655 train_time:18.360s step_avg:146.88ms
step:250/3375 val_loss:4.10478 train_time:36.695s step_avg:146.69ms
step:375/3375 val_loss:3.92713 train_time:54.994s step_avg:146.39ms
step:500/3375 val_loss:3.82238 train_time:73.298s step_avg:146.43ms
step:625/3375 val_loss:3.75522 train_time:91.562s step_avg:146.11ms
step:750/3375 val_loss:3.70819 train_time:109.797s step_avg:145.88ms
step:875/3375 val_loss:3.66502 train_time:128.079s step_avg:146.26ms
step:1000/3375 val_loss:3.62934 train_time:146.359s step_avg:146.24ms
step:1125/3375 val_loss:3.60386 train_time:164.619s step_avg:146.08ms
step:1250/3375 val_loss:3.57088 train_time:182.898s step_avg:146.24ms
step:1375/3375 val_loss:3.54478 train_time:201.181s step_avg:146.26ms
step:1500/3375 val_loss:3.51794 train_time:219.442s step_avg:146.09ms
step:1625/3375 val_loss:3.49826 train_time:237.781s step_avg:146.71ms
step:1750/3375 val_loss:3.47474 train_time:256.054s step_avg:146.18ms
step:1875/3375 val_loss:3.45550 train_time:274.316s step_avg:146.10ms
step:2000/3375 val_loss:3.43605 train_time:292.589s step_avg:146.18ms
step:2125/3375 val_loss:3.41939 train_time:310.853s step_avg:146.12ms
step:2250/3375 val_loss:3.40269 train_time:329.111s step_avg:146.06ms
step:2375/3375 val_loss:3.38718 train_time:347.385s step_avg:146.19ms
step:2500/3375 val_loss:3.37142 train_time:365.667s step_avg:146.26ms
step:2625/3375 val_loss:3.35572 train_time:383.924s step_avg:146.05ms
step:2750/3375 val_loss:3.34083 train_time:402.200s step_avg:146.20ms
step:2875/3375 val_loss:3.32631 train_time:420.477s step_avg:146.22ms
step:3000/3375 val_loss:3.31205 train_time:438.724s step_avg:145.98ms
step:3125/3375 val_loss:3.29850 train_time:457.008s step_avg:146.27ms
step:3250/3375 val_loss:3.28707 train_time:475.281s step_avg:146.18ms
step:3375/3375 val_loss:3.28106 train_time:493.532s step_avg:146.01ms
step:0/3375 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3375 val_loss:4.63458 train_time:18.361s step_avg:146.89ms
step:250/3375 val_loss:4.10816 train_time:36.690s step_avg:146.63ms
step:375/3375 val_loss:3.92496 train_time:54.963s step_avg:146.18ms
step:500/3375 val_loss:3.82368 train_time:73.254s step_avg:146.32ms
step:625/3375 val_loss:3.75178 train_time:91.532s step_avg:146.23ms
step:750/3375 val_loss:3.70479 train_time:109.786s step_avg:146.03ms
step:875/3375 val_loss:3.66472 train_time:128.067s step_avg:146.24ms
step:1000/3375 val_loss:3.62945 train_time:146.356s step_avg:146.32ms
step:1125/3375 val_loss:3.60192 train_time:164.611s step_avg:146.03ms
step:1250/3375 val_loss:3.56665 train_time:182.887s step_avg:146.21ms
step:1375/3375 val_loss:3.54214 train_time:201.172s step_avg:146.28ms
step:1500/3375 val_loss:3.51459 train_time:219.431s step_avg:146.07ms
step:1625/3375 val_loss:3.49360 train_time:237.715s step_avg:146.27ms
step:1750/3375 val_loss:3.47193 train_time:255.992s step_avg:146.22ms
step:1875/3375 val_loss:3.45164 train_time:274.242s step_avg:146.00ms
step:2000/3375 val_loss:3.43326 train_time:292.526s step_avg:146.27ms
step:2125/3375 val_loss:3.41626 train_time:310.810s step_avg:146.27ms
step:2250/3375 val_loss:3.39934 train_time:329.141s step_avg:146.64ms
step:2375/3375 val_loss:3.38413 train_time:347.411s step_avg:146.16ms
step:2500/3375 val_loss:3.36901 train_time:365.702s step_avg:146.33ms
step:2625/3375 val_loss:3.35265 train_time:383.962s step_avg:146.08ms
step:2750/3375 val_loss:3.33852 train_time:402.244s step_avg:146.26ms
step:2875/3375 val_loss:3.32370 train_time:420.521s step_avg:146.22ms
step:3000/3375 val_loss:3.30955 train_time:438.787s step_avg:146.12ms
step:3125/3375 val_loss:3.29588 train_time:457.064s step_avg:146.22ms
step:3250/3375 val_loss:3.28454 train_time:475.346s step_avg:146.25ms
step:3375/3375 val_loss:3.27843 train_time:493.618s step_avg:146.18ms
step:0/3375 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3375 val_loss:4.65295 train_time:18.372s step_avg:146.97ms
step:250/3375 val_loss:4.11288 train_time:36.709s step_avg:146.70ms
step:375/3375 val_loss:3.93115 train_time:55.000s step_avg:146.32ms
step:500/3375 val_loss:3.82625 train_time:73.299s step_avg:146.39ms
step:625/3375 val_loss:3.75756 train_time:91.583s step_avg:146.28ms
step:750/3375 val_loss:3.70831 train_time:109.842s step_avg:146.07ms
step:875/3375 val_loss:3.66657 train_time:128.123s step_avg:146.25ms
step:1000/3375 val_loss:3.63211 train_time:146.419s step_avg:146.37ms
step:1125/3375 val_loss:3.60464 train_time:164.683s step_avg:146.12ms
step:1250/3375 val_loss:3.57049 train_time:182.965s step_avg:146.25ms
step:1375/3375 val_loss:3.54572 train_time:201.245s step_avg:146.24ms
step:1500/3375 val_loss:3.51626 train_time:219.493s step_avg:145.98ms
step:1625/3375 val_loss:3.49755 train_time:237.783s step_avg:146.32ms
step:1750/3375 val_loss:3.47460 train_time:256.072s step_avg:146.31ms
step:1875/3375 val_loss:3.45433 train_time:274.330s step_avg:146.07ms
step:2000/3375 val_loss:3.43550 train_time:292.605s step_avg:146.20ms
step:2125/3375 val_loss:3.41852 train_time:310.880s step_avg:146.19ms
step:2250/3375 val_loss:3.40150 train_time:329.130s step_avg:146.00ms
step:2375/3375 val_loss:3.38614 train_time:347.411s step_avg:146.25ms
step:2500/3375 val_loss:3.37027 train_time:365.676s step_avg:146.11ms
step:2625/3375 val_loss:3.35458 train_time:383.949s step_avg:146.19ms
step:2750/3375 val_loss:3.33972 train_time:402.306s step_avg:146.85ms
step:2875/3375 val_loss:3.32545 train_time:420.577s step_avg:146.18ms
step:3000/3375 val_loss:3.31085 train_time:438.832s step_avg:146.03ms
step:3125/3375 val_loss:3.29736 train_time:457.113s step_avg:146.25ms
step:3250/3375 val_loss:3.28604 train_time:475.385s step_avg:146.18ms
step:3375/3375 val_loss:3.27995 train_time:493.646s step_avg:146.09ms
step:0/3375 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3375 val_loss:4.64621 train_time:18.362s step_avg:146.90ms
step:250/3375 val_loss:4.11431 train_time:36.696s step_avg:146.67ms
step:375/3375 val_loss:3.92784 train_time:54.992s step_avg:146.37ms
step:500/3375 val_loss:3.82071 train_time:73.298s step_avg:146.45ms
step:625/3375 val_loss:3.75224 train_time:91.594s step_avg:146.37ms
step:750/3375 val_loss:3.70740 train_time:109.861s step_avg:146.14ms
step:875/3375 val_loss:3.66244 train_time:128.136s step_avg:146.20ms
step:1000/3375 val_loss:3.62866 train_time:146.420s step_avg:146.27ms
step:1125/3375 val_loss:3.59918 train_time:164.674s step_avg:146.04ms
step:1250/3375 val_loss:3.56743 train_time:182.953s step_avg:146.23ms
step:1375/3375 val_loss:3.54093 train_time:201.230s step_avg:146.21ms
step:1500/3375 val_loss:3.51239 train_time:219.482s step_avg:146.02ms
step:1625/3375 val_loss:3.49353 train_time:237.768s step_avg:146.29ms
step:1750/3375 val_loss:3.47097 train_time:256.040s step_avg:146.18ms
step:1875/3375 val_loss:3.45120 train_time:274.293s step_avg:146.02ms
step:2000/3375 val_loss:3.43291 train_time:292.589s step_avg:146.37ms
step:2125/3375 val_loss:3.41565 train_time:310.870s step_avg:146.25ms
step:2250/3375 val_loss:3.39906 train_time:329.135s step_avg:146.12ms
step:2375/3375 val_loss:3.38363 train_time:347.418s step_avg:146.26ms
step:2500/3375 val_loss:3.36834 train_time:365.717s step_avg:146.40ms
step:2625/3375 val_loss:3.35229 train_time:383.968s step_avg:146.00ms
step:2750/3375 val_loss:3.33766 train_time:402.260s step_avg:146.34ms
step:2875/3375 val_loss:3.32342 train_time:420.539s step_avg:146.23ms
step:3000/3375 val_loss:3.30923 train_time:438.794s step_avg:146.04ms
step:3125/3375 val_loss:3.29548 train_time:457.080s step_avg:146.29ms
step:3250/3375 val_loss:3.28406 train_time:475.431s step_avg:146.81ms
step:3375/3375 val_loss:3.27806 train_time:493.684s step_avg:146.03ms
step:0/3375 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3375 val_loss:4.64746 train_time:18.368s step_avg:146.94ms
step:250/3375 val_loss:4.10943 train_time:36.703s step_avg:146.68ms
step:375/3375 val_loss:3.92966 train_time:54.999s step_avg:146.37ms
step:500/3375 val_loss:3.82562 train_time:73.306s step_avg:146.45ms
step:625/3375 val_loss:3.75923 train_time:91.607s step_avg:146.41ms
step:750/3375 val_loss:3.71092 train_time:109.877s step_avg:146.16ms
step:875/3375 val_loss:3.66742 train_time:128.163s step_avg:146.29ms
step:1000/3375 val_loss:3.63041 train_time:146.446s step_avg:146.27ms
step:1125/3375 val_loss:3.60497 train_time:164.701s step_avg:146.04ms
step:1250/3375 val_loss:3.57122 train_time:182.986s step_avg:146.28ms
step:1375/3375 val_loss:3.54525 train_time:201.264s step_avg:146.22ms
step:1500/3375 val_loss:3.51711 train_time:219.523s step_avg:146.07ms
step:1625/3375 val_loss:3.49729 train_time:237.800s step_avg:146.22ms
step:1750/3375 val_loss:3.47538 train_time:256.083s step_avg:146.26ms
step:1875/3375 val_loss:3.45492 train_time:274.343s step_avg:146.08ms
step:2000/3375 val_loss:3.43663 train_time:292.615s step_avg:146.18ms
step:2125/3375 val_loss:3.41990 train_time:310.886s step_avg:146.17ms
step:2250/3375 val_loss:3.40277 train_time:329.134s step_avg:145.99ms
step:2375/3375 val_loss:3.38730 train_time:347.406s step_avg:146.17ms
step:2500/3375 val_loss:3.37208 train_time:365.668s step_avg:146.10ms
step:2625/3375 val_loss:3.35596 train_time:383.923s step_avg:146.04ms
step:2750/3375 val_loss:3.34114 train_time:402.194s step_avg:146.17ms
step:2875/3375 val_loss:3.32696 train_time:420.461s step_avg:146.13ms
step:3000/3375 val_loss:3.31272 train_time:438.719s step_avg:146.06ms
step:3125/3375 val_loss:3.29894 train_time:456.994s step_avg:146.21ms
step:3250/3375 val_loss:3.28774 train_time:475.279s step_avg:146.28ms
step:3375/3375 val_loss:3.28161 train_time:493.535s step_avg:146.05ms
step:0/3375 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3375 val_loss:4.64823 train_time:18.367s step_avg:146.93ms
step:250/3375 val_loss:4.10896 train_time:36.690s step_avg:146.59ms
step:375/3375 val_loss:3.92366 train_time:55.043s step_avg:146.82ms
step:500/3375 val_loss:3.82054 train_time:73.339s step_avg:146.37ms
step:625/3375 val_loss:3.75112 train_time:91.628s step_avg:146.31ms
step:750/3375 val_loss:3.70325 train_time:109.897s step_avg:146.15ms
step:875/3375 val_loss:3.66357 train_time:128.185s step_avg:146.31ms
step:1000/3375 val_loss:3.62859 train_time:146.474s step_avg:146.31ms
step:1125/3375 val_loss:3.59966 train_time:164.704s step_avg:145.84ms
step:1250/3375 val_loss:3.56645 train_time:183.000s step_avg:146.37ms
step:1375/3375 val_loss:3.54045 train_time:201.282s step_avg:146.26ms
step:1500/3375 val_loss:3.51307 train_time:219.549s step_avg:146.14ms
step:1625/3375 val_loss:3.49296 train_time:237.832s step_avg:146.26ms
step:1750/3375 val_loss:3.47182 train_time:256.118s step_avg:146.29ms
step:1875/3375 val_loss:3.45162 train_time:274.376s step_avg:146.06ms
step:2000/3375 val_loss:3.43209 train_time:292.665s step_avg:146.31ms
step:2125/3375 val_loss:3.41591 train_time:310.952s step_avg:146.30ms
step:2250/3375 val_loss:3.39856 train_time:329.212s step_avg:146.08ms
step:2375/3375 val_loss:3.38317 train_time:347.495s step_avg:146.27ms
step:2500/3375 val_loss:3.36787 train_time:365.785s step_avg:146.32ms
step:2625/3375 val_loss:3.35197 train_time:384.057s step_avg:146.17ms
step:2750/3375 val_loss:3.33718 train_time:402.345s step_avg:146.31ms
step:2875/3375 val_loss:3.32248 train_time:420.629s step_avg:146.27ms
step:3000/3375 val_loss:3.30872 train_time:438.895s step_avg:146.13ms
step:3125/3375 val_loss:3.29495 train_time:457.181s step_avg:146.29ms
step:3250/3375 val_loss:3.28373 train_time:475.460s step_avg:146.23ms
step:3375/3375 val_loss:3.27765 train_time:493.726s step_avg:146.13ms
step:0/3375 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3375 val_loss:4.61266 train_time:18.342s step_avg:146.73ms
step:250/3375 val_loss:4.10404 train_time:36.668s step_avg:146.61ms
step:375/3375 val_loss:3.92379 train_time:54.965s step_avg:146.38ms
step:500/3375 val_loss:3.82165 train_time:73.259s step_avg:146.35ms
step:625/3375 val_loss:3.75049 train_time:91.554s step_avg:146.36ms
step:750/3375 val_loss:3.70373 train_time:109.810s step_avg:146.05ms
step:875/3375 val_loss:3.66445 train_time:128.168s step_avg:146.86ms
step:1000/3375 val_loss:3.62908 train_time:146.436s step_avg:146.14ms
step:1125/3375 val_loss:3.59977 train_time:164.664s step_avg:145.82ms
step:1250/3375 val_loss:3.56772 train_time:182.928s step_avg:146.12ms
step:1375/3375 val_loss:3.54210 train_time:201.173s step_avg:145.96ms
step:1500/3375 val_loss:3.51608 train_time:219.388s step_avg:145.72ms
step:1625/3375 val_loss:3.49623 train_time:237.666s step_avg:146.23ms
step:1750/3375 val_loss:3.47328 train_time:255.935s step_avg:146.15ms
step:1875/3375 val_loss:3.45353 train_time:274.194s step_avg:146.07ms
step:2000/3375 val_loss:3.43378 train_time:292.471s step_avg:146.22ms
step:2125/3375 val_loss:3.41760 train_time:310.748s step_avg:146.22ms
step:2250/3375 val_loss:3.40041 train_time:329.005s step_avg:146.05ms
step:2375/3375 val_loss:3.38494 train_time:347.289s step_avg:146.27ms
step:2500/3375 val_loss:3.36980 train_time:365.567s step_avg:146.23ms
step:2625/3375 val_loss:3.35389 train_time:383.823s step_avg:146.05ms
step:2750/3375 val_loss:3.33899 train_time:402.103s step_avg:146.23ms
step:2875/3375 val_loss:3.32444 train_time:420.381s step_avg:146.22ms
step:3000/3375 val_loss:3.31071 train_time:438.630s step_avg:146.00ms
step:3125/3375 val_loss:3.29687 train_time:456.921s step_avg:146.33ms
step:3250/3375 val_loss:3.28552 train_time:475.206s step_avg:146.28ms
step:3375/3375 val_loss:3.27945 train_time:493.474s step_avg:146.15ms
step:0/3375 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3375 val_loss:4.64940 train_time:18.361s step_avg:146.89ms
step:250/3375 val_loss:4.10884 train_time:36.680s step_avg:146.55ms
step:375/3375 val_loss:3.92730 train_time:54.970s step_avg:146.32ms
step:500/3375 val_loss:3.82125 train_time:73.258s step_avg:146.30ms
step:625/3375 val_loss:3.75152 train_time:91.547s step_avg:146.32ms
step:750/3375 val_loss:3.70359 train_time:109.817s step_avg:146.15ms
step:875/3375 val_loss:3.66367 train_time:128.083s step_avg:146.13ms
step:1000/3375 val_loss:3.62863 train_time:146.368s step_avg:146.29ms
step:1125/3375 val_loss:3.60173 train_time:164.620s step_avg:146.01ms
step:1250/3375 val_loss:3.56803 train_time:182.895s step_avg:146.20ms
step:1375/3375 val_loss:3.54235 train_time:201.176s step_avg:146.24ms
step:1500/3375 val_loss:3.51403 train_time:219.491s step_avg:146.52ms
step:1625/3375 val_loss:3.49391 train_time:237.766s step_avg:146.20ms
step:1750/3375 val_loss:3.47204 train_time:256.038s step_avg:146.17ms
step:1875/3375 val_loss:3.45285 train_time:274.282s step_avg:145.95ms
step:2000/3375 val_loss:3.43393 train_time:292.540s step_avg:146.07ms
step:2125/3375 val_loss:3.41730 train_time:310.796s step_avg:146.04ms
step:2250/3375 val_loss:3.40060 train_time:329.035s step_avg:145.91ms
step:2375/3375 val_loss:3.38508 train_time:347.287s step_avg:146.01ms
step:2500/3375 val_loss:3.37000 train_time:365.535s step_avg:145.98ms
step:2625/3375 val_loss:3.35382 train_time:383.772s step_avg:145.90ms
step:2750/3375 val_loss:3.33890 train_time:402.038s step_avg:146.13ms
step:2875/3375 val_loss:3.32477 train_time:420.301s step_avg:146.10ms
step:3000/3375 val_loss:3.31071 train_time:438.535s step_avg:145.87ms
step:3125/3375 val_loss:3.29670 train_time:456.789s step_avg:146.03ms
step:3250/3375 val_loss:3.28548 train_time:475.051s step_avg:146.10ms
step:3375/3375 val_loss:3.27936 train_time:493.301s step_avg:146.00ms
step:0/3375 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3375 val_loss:4.64565 train_time:18.351s step_avg:146.80ms
step:250/3375 val_loss:4.11468 train_time:36.657s step_avg:146.45ms
step:375/3375 val_loss:3.92997 train_time:54.938s step_avg:146.25ms
step:500/3375 val_loss:3.82522 train_time:73.214s step_avg:146.21ms
step:625/3375 val_loss:3.75494 train_time:91.490s step_avg:146.20ms
step:750/3375 val_loss:3.70924 train_time:109.741s step_avg:146.01ms
step:875/3375 val_loss:3.66545 train_time:128.018s step_avg:146.22ms
step:1000/3375 val_loss:3.63133 train_time:146.290s step_avg:146.18ms
step:1125/3375 val_loss:3.60351 train_time:164.544s step_avg:146.03ms
step:1250/3375 val_loss:3.57226 train_time:182.796s step_avg:146.02ms
step:1375/3375 val_loss:3.54481 train_time:201.038s step_avg:145.93ms
step:1500/3375 val_loss:3.51888 train_time:219.297s step_avg:146.07ms
step:1625/3375 val_loss:3.49779 train_time:237.562s step_avg:146.12ms
step:1750/3375 val_loss:3.47596 train_time:255.834s step_avg:146.17ms
step:1875/3375 val_loss:3.45614 train_time:274.078s step_avg:145.96ms
step:2000/3375 val_loss:3.43672 train_time:292.421s step_avg:146.74ms
step:2125/3375 val_loss:3.41989 train_time:310.665s step_avg:145.95ms
step:2250/3375 val_loss:3.40341 train_time:328.876s step_avg:145.69ms
step:2375/3375 val_loss:3.38759 train_time:347.111s step_avg:145.88ms
step:2500/3375 val_loss:3.37189 train_time:365.372s step_avg:146.09ms
step:2625/3375 val_loss:3.35614 train_time:383.576s step_avg:145.63ms
step:2750/3375 val_loss:3.34139 train_time:401.808s step_avg:145.85ms
step:2875/3375 val_loss:3.32688 train_time:420.041s step_avg:145.86ms
step:3000/3375 val_loss:3.31236 train_time:438.274s step_avg:145.86ms
step:3125/3375 val_loss:3.29874 train_time:456.524s step_avg:146.00ms
step:3250/3375 val_loss:3.28738 train_time:474.784s step_avg:146.07ms
step:3375/3375 val_loss:3.28124 train_time:493.003s step_avg:145.76ms
step:0/3375 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3375 val_loss:4.63499 train_time:18.346s step_avg:146.77ms
step:250/3375 val_loss:4.10844 train_time:36.647s step_avg:146.41ms
step:375/3375 val_loss:3.92603 train_time:54.907s step_avg:146.08ms
step:500/3375 val_loss:3.82223 train_time:73.171s step_avg:146.11ms
step:625/3375 val_loss:3.75350 train_time:91.420s step_avg:145.99ms
step:750/3375 val_loss:3.70410 train_time:109.671s step_avg:146.01ms
step:875/3375 val_loss:3.66359 train_time:127.920s step_avg:145.99ms
step:1000/3375 val_loss:3.62869 train_time:146.169s step_avg:145.99ms
step:1125/3375 val_loss:3.60080 train_time:164.401s step_avg:145.85ms
step:1250/3375 val_loss:3.56827 train_time:182.657s step_avg:146.05ms
step:1375/3375 val_loss:3.54100 train_time:200.907s step_avg:146.00ms
step:1500/3375 val_loss:3.51468 train_time:219.115s step_avg:145.66ms
step:1625/3375 val_loss:3.49300 train_time:237.367s step_avg:146.02ms
step:1750/3375 val_loss:3.47176 train_time:255.604s step_avg:145.90ms
step:1875/3375 val_loss:3.45215 train_time:273.831s step_avg:145.81ms
step:2000/3375 val_loss:3.43308 train_time:292.100s step_avg:146.15ms
step:2125/3375 val_loss:3.41611 train_time:310.382s step_avg:146.26ms
step:2250/3375 val_loss:3.39898 train_time:328.643s step_avg:146.09ms
step:2375/3375 val_loss:3.38369 train_time:346.923s step_avg:146.24ms
step:2500/3375 val_loss:3.36826 train_time:365.273s step_avg:146.80ms
step:2625/3375 val_loss:3.35261 train_time:383.527s step_avg:146.03ms
step:2750/3375 val_loss:3.33779 train_time:401.815s step_avg:146.31ms
step:2875/3375 val_loss:3.32344 train_time:420.097s step_avg:146.26ms
step:3000/3375 val_loss:3.30914 train_time:438.334s step_avg:145.90ms
step:3125/3375 val_loss:3.29541 train_time:456.578s step_avg:145.95ms
step:3250/3375 val_loss:3.28414 train_time:474.838s step_avg:146.08ms
step:3375/3375 val_loss:3.27804 train_time:493.079s step_avg:145.93ms
step:0/3375 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3375 val_loss:4.64418 train_time:18.344s step_avg:146.75ms
step:250/3375 val_loss:4.10495 train_time:36.644s step_avg:146.40ms
step:375/3375 val_loss:3.92620 train_time:54.916s step_avg:146.18ms
step:500/3375 val_loss:3.82128 train_time:73.215s step_avg:146.39ms
step:625/3375 val_loss:3.75060 train_time:91.494s step_avg:146.23ms
step:750/3375 val_loss:3.70437 train_time:109.760s step_avg:146.13ms
step:875/3375 val_loss:3.66335 train_time:128.053s step_avg:146.35ms
step:1000/3375 val_loss:3.62684 train_time:146.343s step_avg:146.32ms
step:1125/3375 val_loss:3.60003 train_time:164.605s step_avg:146.09ms
step:1250/3375 val_loss:3.56894 train_time:182.892s step_avg:146.30ms
step:1375/3375 val_loss:3.54121 train_time:201.179s step_avg:146.29ms
step:1500/3375 val_loss:3.51337 train_time:219.439s step_avg:146.08ms
step:1625/3375 val_loss:3.49317 train_time:237.712s step_avg:146.18ms
step:1750/3375 val_loss:3.47153 train_time:255.992s step_avg:146.24ms
step:1875/3375 val_loss:3.45234 train_time:274.254s step_avg:146.10ms
step:2000/3375 val_loss:3.43337 train_time:292.516s step_avg:146.09ms
step:2125/3375 val_loss:3.41669 train_time:310.784s step_avg:146.15ms
step:2250/3375 val_loss:3.39979 train_time:329.049s step_avg:146.12ms
step:2375/3375 val_loss:3.38414 train_time:347.346s step_avg:146.38ms
step:2500/3375 val_loss:3.36889 train_time:365.629s step_avg:146.26ms
step:2625/3375 val_loss:3.35304 train_time:383.885s step_avg:146.05ms
step:2750/3375 val_loss:3.33838 train_time:402.180s step_avg:146.36ms
step:2875/3375 val_loss:3.32379 train_time:420.465s step_avg:146.28ms
step:3000/3375 val_loss:3.30969 train_time:438.785s step_avg:146.56ms
step:3125/3375 val_loss:3.29601 train_time:457.073s step_avg:146.31ms
step:3250/3375 val_loss:3.28466 train_time:475.354s step_avg:146.24ms
step:3375/3375 val_loss:3.27859 train_time:493.620s step_avg:146.13ms
step:0/3375 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3375 val_loss:4.64425 train_time:18.376s step_avg:147.01ms
step:250/3375 val_loss:4.10806 train_time:36.711s step_avg:146.68ms
step:375/3375 val_loss:3.92875 train_time:55.009s step_avg:146.38ms
step:500/3375 val_loss:3.82096 train_time:73.317s step_avg:146.46ms
step:625/3375 val_loss:3.75253 train_time:91.619s step_avg:146.42ms
step:750/3375 val_loss:3.70463 train_time:109.908s step_avg:146.31ms
step:875/3375 val_loss:3.66251 train_time:128.208s step_avg:146.41ms
step:1000/3375 val_loss:3.62759 train_time:146.517s step_avg:146.47ms
step:1125/3375 val_loss:3.60102 train_time:164.790s step_avg:146.18ms
step:1250/3375 val_loss:3.56771 train_time:183.085s step_avg:146.36ms
step:1375/3375 val_loss:3.54059 train_time:201.378s step_avg:146.34ms
step:1500/3375 val_loss:3.51082 train_time:219.649s step_avg:146.17ms
step:1625/3375 val_loss:3.49172 train_time:237.940s step_avg:146.33ms
step:1750/3375 val_loss:3.47055 train_time:256.239s step_avg:146.39ms
step:1875/3375 val_loss:3.45131 train_time:274.507s step_avg:146.15ms
step:2000/3375 val_loss:3.43258 train_time:292.796s step_avg:146.31ms
step:2125/3375 val_loss:3.41605 train_time:311.096s step_avg:146.41ms
step:2250/3375 val_loss:3.39902 train_time:329.358s step_avg:146.10ms
step:2375/3375 val_loss:3.38343 train_time:347.652s step_avg:146.35ms
step:2500/3375 val_loss:3.36815 train_time:365.939s step_avg:146.30ms
step:2625/3375 val_loss:3.35207 train_time:384.203s step_avg:146.11ms
step:2750/3375 val_loss:3.33726 train_time:402.497s step_avg:146.35ms
step:2875/3375 val_loss:3.32291 train_time:420.782s step_avg:146.28ms
step:3000/3375 val_loss:3.30871 train_time:439.048s step_avg:146.13ms
step:3125/3375 val_loss:3.29490 train_time:457.331s step_avg:146.26ms
step:3250/3375 val_loss:3.28358 train_time:475.628s step_avg:146.37ms
step:3375/3375 val_loss:3.27754 train_time:493.909s step_avg:146.25ms
step:0/3375 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3375 val_loss:4.62710 train_time:18.439s step_avg:147.51ms
step:250/3375 val_loss:4.10570 train_time:36.780s step_avg:146.73ms
step:375/3375 val_loss:3.92416 train_time:55.075s step_avg:146.36ms
step:500/3375 val_loss:3.82192 train_time:73.387s step_avg:146.49ms
step:625/3375 val_loss:3.75160 train_time:91.694s step_avg:146.46ms
step:750/3375 val_loss:3.70375 train_time:109.977s step_avg:146.26ms
step:875/3375 val_loss:3.66345 train_time:128.276s step_avg:146.39ms
step:1000/3375 val_loss:3.62761 train_time:146.566s step_avg:146.32ms
step:1125/3375 val_loss:3.59991 train_time:164.834s step_avg:146.14ms
step:1250/3375 val_loss:3.56718 train_time:183.127s step_avg:146.35ms
step:1375/3375 val_loss:3.54105 train_time:201.423s step_avg:146.37ms
step:1500/3375 val_loss:3.51138 train_time:219.699s step_avg:146.20ms
step:1625/3375 val_loss:3.49234 train_time:238.003s step_avg:146.43ms
step:1750/3375 val_loss:3.47088 train_time:256.304s step_avg:146.41ms
step:1875/3375 val_loss:3.45141 train_time:274.583s step_avg:146.24ms
step:2000/3375 val_loss:3.43222 train_time:292.878s step_avg:146.36ms
step:2125/3375 val_loss:3.41544 train_time:311.168s step_avg:146.32ms
step:2250/3375 val_loss:3.39871 train_time:329.447s step_avg:146.23ms
step:2375/3375 val_loss:3.38319 train_time:347.749s step_avg:146.42ms
step:2500/3375 val_loss:3.36759 train_time:366.041s step_avg:146.33ms
step:2625/3375 val_loss:3.35178 train_time:384.325s step_avg:146.27ms
step:2750/3375 val_loss:3.33675 train_time:402.622s step_avg:146.38ms
step:2875/3375 val_loss:3.32256 train_time:420.931s step_avg:146.47ms
step:3000/3375 val_loss:3.30828 train_time:439.215s step_avg:146.27ms
step:3125/3375 val_loss:3.29473 train_time:457.517s step_avg:146.41ms
step:3250/3375 val_loss:3.28346 train_time:475.814s step_avg:146.38ms
step:3375/3375 val_loss:3.27728 train_time:494.088s step_avg:146.19ms
step:0/3375 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3375 val_loss:4.63480 train_time:18.352s step_avg:146.82ms
step:250/3375 val_loss:4.10846 train_time:36.673s step_avg:146.57ms
step:375/3375 val_loss:3.92466 train_time:54.979s step_avg:146.45ms
step:500/3375 val_loss:3.82300 train_time:73.310s step_avg:146.65ms
step:625/3375 val_loss:3.75082 train_time:91.613s step_avg:146.42ms
step:750/3375 val_loss:3.70473 train_time:109.926s step_avg:146.51ms
step:875/3375 val_loss:3.66237 train_time:128.210s step_avg:146.27ms
step:1000/3375 val_loss:3.62833 train_time:146.480s step_avg:146.16ms
step:1125/3375 val_loss:3.60009 train_time:164.732s step_avg:146.02ms
step:1250/3375 val_loss:3.56879 train_time:183.002s step_avg:146.16ms
step:1375/3375 val_loss:3.54171 train_time:201.297s step_avg:146.35ms
step:1500/3375 val_loss:3.51398 train_time:219.570s step_avg:146.18ms
step:1625/3375 val_loss:3.49242 train_time:237.828s step_avg:146.07ms
step:1750/3375 val_loss:3.47142 train_time:256.102s step_avg:146.19ms
step:1875/3375 val_loss:3.45036 train_time:274.340s step_avg:145.90ms
step:2000/3375 val_loss:3.43276 train_time:292.626s step_avg:146.29ms
step:2125/3375 val_loss:3.41660 train_time:310.894s step_avg:146.15ms
step:2250/3375 val_loss:3.39883 train_time:329.182s step_avg:146.30ms
step:2375/3375 val_loss:3.38406 train_time:347.479s step_avg:146.38ms
step:2500/3375 val_loss:3.36860 train_time:365.763s step_avg:146.27ms
step:2625/3375 val_loss:3.35244 train_time:384.009s step_avg:145.97ms
step:2750/3375 val_loss:3.33757 train_time:402.278s step_avg:146.15ms
step:2875/3375 val_loss:3.32349 train_time:420.562s step_avg:146.28ms
step:3000/3375 val_loss:3.30922 train_time:438.825s step_avg:146.10ms
step:3125/3375 val_loss:3.29545 train_time:457.105s step_avg:146.24ms
step:3250/3375 val_loss:3.28415 train_time:475.399s step_avg:146.36ms
step:3375/3375 val_loss:3.27802 train_time:493.668s step_avg:146.15ms
step:0/3375 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3375 val_loss:4.62939 train_time:18.394s step_avg:147.16ms
step:250/3375 val_loss:4.10591 train_time:36.735s step_avg:146.72ms
step:375/3375 val_loss:3.92435 train_time:55.036s step_avg:146.40ms
step:500/3375 val_loss:3.82122 train_time:73.349s step_avg:146.50ms
step:625/3375 val_loss:3.75232 train_time:91.672s step_avg:146.58ms
step:750/3375 val_loss:3.70376 train_time:109.954s step_avg:146.26ms
step:875/3375 val_loss:3.66178 train_time:128.261s step_avg:146.46ms
step:1000/3375 val_loss:3.62847 train_time:146.566s step_avg:146.44ms
step:1125/3375 val_loss:3.59896 train_time:164.853s step_avg:146.30ms
step:1250/3375 val_loss:3.56753 train_time:183.220s step_avg:146.93ms
step:1375/3375 val_loss:3.54174 train_time:201.512s step_avg:146.33ms
step:1500/3375 val_loss:3.51301 train_time:219.784s step_avg:146.17ms
step:1625/3375 val_loss:3.49440 train_time:238.079s step_avg:146.36ms
step:1750/3375 val_loss:3.47148 train_time:256.377s step_avg:146.39ms
step:1875/3375 val_loss:3.45219 train_time:274.656s step_avg:146.24ms
step:2000/3375 val_loss:3.43340 train_time:292.954s step_avg:146.38ms
step:2125/3375 val_loss:3.41659 train_time:311.243s step_avg:146.31ms
step:2250/3375 val_loss:3.39970 train_time:329.514s step_avg:146.17ms
step:2375/3375 val_loss:3.38453 train_time:347.799s step_avg:146.28ms
step:2500/3375 val_loss:3.36881 train_time:366.089s step_avg:146.32ms
step:2625/3375 val_loss:3.35298 train_time:384.357s step_avg:146.14ms
step:2750/3375 val_loss:3.33836 train_time:402.637s step_avg:146.24ms
step:2875/3375 val_loss:3.32407 train_time:420.929s step_avg:146.33ms
step:3000/3375 val_loss:3.30980 train_time:439.187s step_avg:146.07ms
step:3125/3375 val_loss:3.29615 train_time:457.479s step_avg:146.33ms
step:3250/3375 val_loss:3.28479 train_time:475.772s step_avg:146.35ms
step:3375/3375 val_loss:3.27877 train_time:494.037s step_avg:146.11ms
step:0/3375 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3375 val_loss:4.65277 train_time:18.378s step_avg:147.03ms
step:250/3375 val_loss:4.11313 train_time:36.713s step_avg:146.68ms
step:375/3375 val_loss:3.92888 train_time:55.008s step_avg:146.36ms
step:500/3375 val_loss:3.82364 train_time:73.316s step_avg:146.47ms
step:625/3375 val_loss:3.75467 train_time:91.617s step_avg:146.40ms
step:750/3375 val_loss:3.70637 train_time:109.892s step_avg:146.20ms
step:875/3375 val_loss:3.66476 train_time:128.171s step_avg:146.23ms
step:1000/3375 val_loss:3.63031 train_time:146.465s step_avg:146.36ms
step:1125/3375 val_loss:3.60255 train_time:164.737s step_avg:146.17ms
step:1250/3375 val_loss:3.57019 train_time:183.026s step_avg:146.32ms
step:1375/3375 val_loss:3.54435 train_time:201.320s step_avg:146.35ms
step:1500/3375 val_loss:3.51458 train_time:219.602s step_avg:146.26ms
step:1625/3375 val_loss:3.49595 train_time:237.900s step_avg:146.39ms
step:1750/3375 val_loss:3.47291 train_time:256.281s step_avg:147.04ms
step:1875/3375 val_loss:3.45351 train_time:274.542s step_avg:146.09ms
step:2000/3375 val_loss:3.43503 train_time:292.825s step_avg:146.26ms
step:2125/3375 val_loss:3.41830 train_time:311.110s step_avg:146.28ms
step:2250/3375 val_loss:3.40101 train_time:329.373s step_avg:146.11ms
step:2375/3375 val_loss:3.38601 train_time:347.662s step_avg:146.31ms
step:2500/3375 val_loss:3.37043 train_time:365.951s step_avg:146.31ms
step:2625/3375 val_loss:3.35440 train_time:384.223s step_avg:146.18ms
step:2750/3375 val_loss:3.33985 train_time:402.512s step_avg:146.31ms
step:2875/3375 val_loss:3.32545 train_time:420.806s step_avg:146.35ms
step:3000/3375 val_loss:3.31130 train_time:439.079s step_avg:146.18ms
step:3125/3375 val_loss:3.29750 train_time:457.359s step_avg:146.24ms
step:3250/3375 val_loss:3.28624 train_time:475.655s step_avg:146.37ms
step:3375/3375 val_loss:3.28010 train_time:493.925s step_avg:146.16ms
step:0/3375 val_loss:10.82584 train_time:0.000s step_avg:nanms
step:125/3375 val_loss:4.62531 train_time:18.379s step_avg:147.03ms
step:250/3375 val_loss:4.10638 train_time:36.714s step_avg:146.68ms
step:375/3375 val_loss:3.92514 train_time:55.007s step_avg:146.34ms
step:500/3375 val_loss:3.82062 train_time:73.321s step_avg:146.51ms
step:625/3375 val_loss:3.75230 train_time:91.632s step_avg:146.49ms
step:750/3375 val_loss:3.70525 train_time:109.910s step_avg:146.22ms
step:875/3375 val_loss:3.66222 train_time:128.207s step_avg:146.38ms
step:1000/3375 val_loss:3.62931 train_time:146.512s step_avg:146.44ms
step:1125/3375 val_loss:3.59943 train_time:164.789s step_avg:146.21ms
step:1250/3375 val_loss:3.56750 train_time:183.098s step_avg:146.47ms
step:1375/3375 val_loss:3.54196 train_time:201.397s step_avg:146.39ms
step:1500/3375 val_loss:3.51385 train_time:219.663s step_avg:146.13ms
step:1625/3375 val_loss:3.49278 train_time:237.959s step_avg:146.37ms
step:1750/3375 val_loss:3.47191 train_time:256.250s step_avg:146.32ms
step:1875/3375 val_loss:3.45240 train_time:274.515s step_avg:146.12ms
step:2000/3375 val_loss:3.43308 train_time:292.803s step_avg:146.30ms
step:2125/3375 val_loss:3.41611 train_time:311.103s step_avg:146.41ms
step:2250/3375 val_loss:3.39949 train_time:329.456s step_avg:146.82ms
step:2375/3375 val_loss:3.38363 train_time:347.759s step_avg:146.43ms
step:2500/3375 val_loss:3.36830 train_time:366.045s step_avg:146.29ms
step:2625/3375 val_loss:3.35256 train_time:384.334s step_avg:146.31ms
step:2750/3375 val_loss:3.33768 train_time:402.640s step_avg:146.45ms
step:2875/3375 val_loss:3.32313 train_time:420.944s step_avg:146.43ms
step:3000/3375 val_loss:3.30903 train_time:439.224s step_avg:146.24ms
step:3125/3375 val_loss:3.29527 train_time:457.527s step_avg:146.42ms
step:3250/3375 val_loss:3.28386 train_time:475.837s step_avg:146.48ms
step:3375/3375 val_loss:3.27784 train_time:494.108s step_avg:146.16ms