From 457ecdf7a1e198a370e69bda36c56312a5ee9283 Mon Sep 17 00:00:00 2001
From: Samuel Park <samuel@lambdal.com>
Date: Wed, 12 Jul 2023 16:31:43 -0400
Subject: [PATCH] RTX 4090 laptop results

---
 .../PyTorch_SSD_AMP/12-07-2023_20-08-06.txt   |  51 ++
 .../PyTorch_SSD_AMP/benchmark.para            |   2 +
 .../PyTorch_SSD_FP32/12-07-2023_20-15-34.txt  |  49 ++
 .../PyTorch_SSD_FP32/benchmark.para           |   2 +
 .../12-07-2023_20-20-52.txt                   |  48 ++
 .../benchmark.para                            |   2 +
 .../12-07-2023_20-09-35.txt                   |  33 +
 .../benchmark.para                            |   2 +
 .../12-07-2023_20-07-03.txt                   |  48 ++
 .../benchmark.para                            |   2 +
 .../12-07-2023_19-53-59.txt                   |  34 +
 .../benchmark.para                            |   2 +
 .../PyTorch_gnmt_FP16/12-07-2023_20-18-34.txt | 233 ++++++
 .../PyTorch_gnmt_FP16/benchmark.para          |   2 +
 .../PyTorch_gnmt_FP32/12-07-2023_20-02-53.txt | 267 +++++++
 .../PyTorch_gnmt_FP32/benchmark.para          |   2 +
 .../PyTorch_ncf_FP16/12-07-2023_19-57-50.txt  |  39 +
 .../PyTorch_ncf_FP16/benchmark.para           |   2 +
 .../PyTorch_ncf_FP32/12-07-2023_19-50-49.txt  |  20 +
 .../PyTorch_ncf_FP32/benchmark.para           |   2 +
 .../12-07-2023_19-52-22.txt                   | 217 ++++++
 .../PyTorch_resnet50_AMP/benchmark.para       |   2 +
 .../12-07-2023_20-15-45.txt                   | 217 ++++++
 .../PyTorch_resnet50_FP32/benchmark.para      |   2 +
 .../12-07-2023_19-51-19.txt                   | 173 +++++
 .../PyTorch_tacotron2_FP16/benchmark.para     |   2 +
 .../12-07-2023_20-12-14.txt                   | 170 +++++
 .../PyTorch_tacotron2_FP32/benchmark.para     |   2 +
 .../12-07-2023_20-08-21.txt                   | 202 +++++
 .../benchmark.para                            |   2 +
 .../12-07-2023_20-17-16.txt                   | 183 +++++
 .../benchmark.para                            |   2 +
 .../12-07-2023_20-10-32.txt                   | 202 +++++
 .../benchmark.para                            |   2 +
 .../12-07-2023_20-00-32.txt                   | 185 +++++
 .../benchmark.para                            |   2 +
 .../12-07-2023_19-58-10.txt                   | 709 +++++++++++++++++
 .../PyTorch_waveglow_FP16/benchmark.para      |   2 +
 .../12-07-2023_20-13-13.txt                   | 710 ++++++++++++++++++
 .../PyTorch_waveglow_FP32/benchmark.para      |   2 +
 pytorch/results/4090laptop_v1/summary.txt     |  22 +
 pytorch/results/4090laptop_v1/sys_pytorch.txt |  10 +
 .../config_v1/config_pytorch_4090laptop_v1.sh |   7 +
 43 files changed, 3869 insertions(+)
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_SSD_AMP/12-07-2023_20-08-06.txt
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_SSD_AMP/benchmark.para
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_SSD_FP32/12-07-2023_20-15-34.txt
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_SSD_FP32/benchmark.para
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_bert_base_squad_FP16/12-07-2023_20-20-52.txt
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_bert_base_squad_FP16/benchmark.para
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_bert_base_squad_FP32/12-07-2023_20-09-35.txt
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_bert_base_squad_FP32/benchmark.para
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_bert_large_squad_FP16/12-07-2023_20-07-03.txt
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_bert_large_squad_FP16/benchmark.para
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_bert_large_squad_FP32/12-07-2023_19-53-59.txt
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_bert_large_squad_FP32/benchmark.para
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_gnmt_FP16/12-07-2023_20-18-34.txt
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_gnmt_FP16/benchmark.para
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_gnmt_FP32/12-07-2023_20-02-53.txt
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_gnmt_FP32/benchmark.para
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_ncf_FP16/12-07-2023_19-57-50.txt
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_ncf_FP16/benchmark.para
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_ncf_FP32/12-07-2023_19-50-49.txt
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_ncf_FP32/benchmark.para
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_resnet50_AMP/12-07-2023_19-52-22.txt
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_resnet50_AMP/benchmark.para
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_resnet50_FP32/12-07-2023_20-15-45.txt
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_resnet50_FP32/benchmark.para
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_tacotron2_FP16/12-07-2023_19-51-19.txt
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_tacotron2_FP16/benchmark.para
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_tacotron2_FP32/12-07-2023_20-12-14.txt
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_tacotron2_FP32/benchmark.para
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_transformerxlbase_FP16/12-07-2023_20-08-21.txt
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_transformerxlbase_FP16/benchmark.para
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_transformerxlbase_FP32/12-07-2023_20-17-16.txt
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_transformerxlbase_FP32/benchmark.para
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_transformerxllarge_FP16/12-07-2023_20-10-32.txt
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_transformerxllarge_FP16/benchmark.para
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_transformerxllarge_FP32/12-07-2023_20-00-32.txt
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_transformerxllarge_FP32/benchmark.para
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_waveglow_FP16/12-07-2023_19-58-10.txt
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_waveglow_FP16/benchmark.para
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_waveglow_FP32/12-07-2023_20-13-13.txt
 create mode 100755 pytorch/results/4090laptop_v1/PyTorch_waveglow_FP32/benchmark.para
 create mode 100644 pytorch/results/4090laptop_v1/summary.txt
 create mode 100755 pytorch/results/4090laptop_v1/sys_pytorch.txt
 create mode 100644 pytorch/scripts/config_v1/config_pytorch_4090laptop_v1.sh

diff --git a/pytorch/results/4090laptop_v1/PyTorch_SSD_AMP/12-07-2023_20-08-06.txt b/pytorch/results/4090laptop_v1/PyTorch_SSD_AMP/12-07-2023_20-08-06.txt
new file mode 100755
index 00000000..9ac990be
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_SSD_AMP/12-07-2023_20-08-06.txt
@@ -0,0 +1,51 @@
+/opt/conda/lib/python3.8/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and will be removed in 0.15, please use 'weights' instead.
+  warnings.warn(
+/opt/conda/lib/python3.8/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and will be removed in 0.15. The current behavior is equivalent to passing `weights=ResNet50_Weights.IMAGENET1K_V1`. You can also use `weights=ResNet50_Weights.DEFAULT` to get the most up-to-date weights.
+  warnings.warn(msg)
+Downloading: "https://download.pytorch.org/models/resnet50-0676ba61.pth" to /root/.cache/torch/hub/checkpoints/resnet50-0676ba61.pth
+
  0%|          | 0.00/97.8M [00:00<?, ?B/s]
  1%|▏         | 1.41M/97.8M [00:00<00:07, 14.3MB/s]
  3%|▎         | 2.77M/97.8M [00:00<00:07, 14.0MB/s]
  4%|▍         | 4.19M/97.8M [00:00<00:06, 14.4MB/s]
  6%|▌         | 5.77M/97.8M [00:00<00:06, 14.2MB/s]
  7%|▋         | 7.12M/97.8M [00:00<00:07, 13.4MB/s]
  9%|▉         | 9.00M/97.8M [00:00<00:06, 15.3MB/s]
 11%|█         | 10.9M/97.8M [00:00<00:05, 16.5MB/s]
 13%|█▎        | 12.7M/97.8M [00:00<00:05, 17.4MB/s]
 15%|█▍        | 14.4M/97.8M [00:00<00:04, 17.6MB/s]
 17%|█▋        | 16.3M/97.8M [00:01<00:04, 18.1MB/s]
 18%|█▊        | 18.0M/97.8M [00:01<00:04, 17.6MB/s]
 20%|██        | 19.9M/97.8M [00:01<00:04, 18.1MB/s]
 22%|██▏       | 21.7M/97.8M [00:01<00:04, 18.5MB/s]
 24%|██▍       | 23.5M/97.8M [00:01<00:04, 18.3MB/s]
 26%|██▌       | 25.3M/97.8M [00:01<00:04, 18.0MB/s]
 28%|██▊       | 27.0M/97.8M [00:01<00:04, 17.7MB/s]
 29%|██▉       | 28.8M/97.8M [00:01<00:04, 18.0MB/s]
 31%|███▏      | 30.7M/97.8M [00:01<00:03, 18.5MB/s]
 33%|███▎      | 32.5M/97.8M [00:01<00:03, 18.1MB/s]
 35%|███▌      | 34.2M/97.8M [00:02<00:04, 15.7MB/s]
 37%|███▋      | 35.9M/97.8M [00:02<00:04, 16.1MB/s]
 39%|███▊      | 37.8M/97.8M [00:02<00:03, 17.0MB/s]
 41%|████      | 39.7M/97.8M [00:02<00:03, 17.5MB/s]
 43%|████▎     | 41.6M/97.8M [00:02<00:03, 18.2MB/s]
 45%|████▍     | 43.6M/97.8M [00:02<00:03, 18.7MB/s]
 46%|████▋     | 45.4M/97.8M [00:02<00:02, 18.6MB/s]
 48%|████▊     | 47.2M/97.8M [00:02<00:02, 18.3MB/s]
 50%|█████     | 48.9M/97.8M [00:03<00:03, 16.6MB/s]
 52%|█████▏    | 50.5M/97.8M [00:03<00:03, 15.9MB/s]
 54%|█████▍    | 52.6M/97.8M [00:03<00:02, 17.4MB/s]
 56%|█████▌    | 54.3M/97.8M [00:03<00:02, 17.5MB/s]
 57%|█████▋    | 56.0M/97.8M [00:03<00:02, 17.6MB/s]
 59%|█████▉    | 57.9M/97.8M [00:03<00:02, 18.2MB/s]
 61%|██████    | 59.9M/97.8M [00:03<00:02, 18.9MB/s]
 63%|██████▎   | 61.7M/97.8M [00:03<00:02, 18.5MB/s]
 65%|██████▌   | 63.7M/97.8M [00:03<00:01, 19.1MB/s]
 67%|██████▋   | 65.6M/97.8M [00:03<00:01, 19.3MB/s]
 69%|██████▉   | 67.5M/97.8M [00:04<00:01, 19.6MB/s]
 71%|███████   | 69.4M/97.8M [00:04<00:01, 18.8MB/s]
 73%|███████▎  | 71.2M/97.8M [00:04<00:01, 17.3MB/s]
 75%|███████▍  | 72.9M/97.8M [00:04<00:01, 17.2MB/s]
 76%|███████▋  | 74.6M/97.8M [00:04<00:01, 16.7MB/s]
 78%|███████▊  | 76.2M/97.8M [00:04<00:01, 16.3MB/s]
 80%|███████▉  | 77.7M/97.8M [00:04<00:01, 15.0MB/s]
 81%|████████  | 79.2M/97.8M [00:04<00:01, 15.1MB/s]
 83%|████████▎ | 80.7M/97.8M [00:04<00:01, 15.1MB/s]
 84%|████████▍ | 82.1M/97.8M [00:05<00:01, 15.2MB/s]
 86%|████████▌ | 83.7M/97.8M [00:05<00:00, 15.2MB/s]
 87%|████████▋ | 85.4M/97.8M [00:05<00:00, 15.6MB/s]
 89%|████████▉ | 86.9M/97.8M [00:05<00:00, 15.7MB/s]
 91%|█████████ | 88.5M/97.8M [00:05<00:00, 16.0MB/s]
 92%|█████████▏| 90.0M/97.8M [00:05<00:00, 15.8MB/s]
 94%|█████████▎| 91.6M/97.8M [00:05<00:00, 15.6MB/s]
 95%|█████████▌| 93.1M/97.8M [00:05<00:00, 15.7MB/s]
 97%|█████████▋| 94.6M/97.8M [00:05<00:00, 15.7MB/s]
 98%|█████████▊| 96.2M/97.8M [00:05<00:00, 15.9MB/s]
100%|█████████▉| 97.7M/97.8M [00:06<00:00, 15.9MB/s]
100%|██████████| 97.8M/97.8M [00:06<00:00, 16.9MB/s]
+NOTE! Installing ujson may make loading annotations faster.
+DLL 2023-07-12 20:08:14.147871 - PARAMETER dataset path : /data/object_detection  epochs : 1  batch size : 96  eval batch size : 32  no cuda : False  seed : None  checkpoint path : None  mode : benchmark-training  eval on epochs : [21, 31, 37, 42, 48, 53, 59, 64]  lr decay epochs : [43, 54]  learning rate : 0.0  momentum : 0.9  weight decay : 0.0005  lr warmup : None  backbone : resnet50  backbone path : None  num workers : 4  AMP : True  precision : amp 
+Using seed = 1235
+loading annotations into memory...
+Traceback (most recent call last):
+  File "main.py", line 286, in <module>
+    train(train_loop_func, logger, args)
+  File "main.py", line 146, in train
+    cocoGt = get_coco_ground_truth(args)
+  File "/workspace/benchmark/Detection/SSD/ssd/data.py", line 73, in get_coco_ground_truth
+    cocoGt = COCO(annotation_file=val_annotate, use_ext=True)
+  File "/opt/conda/lib/python3.8/site-packages/pycocotools/coco.py", line 89, in __init__
+    dataset = json.load(open(annotation_file, 'r'))
+FileNotFoundError: [Errno 2] No such file or directory: '/data/object_detection/annotations/instances_val2017.json'
+ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 3160) of binary: /opt/conda/bin/python
+Traceback (most recent call last):
+  File "/opt/conda/bin/torchrun", line 33, in <module>
+    sys.exit(load_entry_point('torch==1.13.0a0+d0d6b1f', 'console_scripts', 'torchrun')())
+  File "/opt/conda/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper
+    return f(*args, **kwargs)
+  File "/opt/conda/lib/python3.8/site-packages/torch/distributed/run.py", line 762, in main
+    run(args)
+  File "/opt/conda/lib/python3.8/site-packages/torch/distributed/run.py", line 753, in run
+    elastic_launch(
+  File "/opt/conda/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 132, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/opt/conda/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent
+    raise ChildFailedError(
+torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
+============================================================
+main.py FAILED
+------------------------------------------------------------
+Failures:
+  <NO_OTHER_FAILURES>
+------------------------------------------------------------
+Root Cause (first observed failure):
+[0]:
+  time      : 2023-07-12_20:08:16
+  host      : 24a2da7181c1
+  rank      : 0 (local_rank: 0)
+  exitcode  : 1 (pid: 3160)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+============================================================
+DONE!
diff --git a/pytorch/results/4090laptop_v1/PyTorch_SSD_AMP/benchmark.para b/pytorch/results/4090laptop_v1/PyTorch_SSD_AMP/benchmark.para
new file mode 100755
index 00000000..70c9b31e
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_SSD_AMP/benchmark.para
@@ -0,0 +1,2 @@
+GLOBAL_BATCH 96
+GPU 1
diff --git a/pytorch/results/4090laptop_v1/PyTorch_SSD_FP32/12-07-2023_20-15-34.txt b/pytorch/results/4090laptop_v1/PyTorch_SSD_FP32/12-07-2023_20-15-34.txt
new file mode 100755
index 00000000..07b0b326
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_SSD_FP32/12-07-2023_20-15-34.txt
@@ -0,0 +1,49 @@
+/opt/conda/lib/python3.8/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and will be removed in 0.15, please use 'weights' instead.
+  warnings.warn(
+/opt/conda/lib/python3.8/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and will be removed in 0.15. The current behavior is equivalent to passing `weights=ResNet50_Weights.IMAGENET1K_V1`. You can also use `weights=ResNet50_Weights.DEFAULT` to get the most up-to-date weights.
+  warnings.warn(msg)
+NOTE! Installing ujson may make loading annotations faster.
+DLL 2023-07-12 20:15:36.177470 - PARAMETER dataset path : /data/object_detection  epochs : 1  batch size : 48  eval batch size : 32  no cuda : False  seed : None  checkpoint path : None  mode : benchmark-training  eval on epochs : [21, 31, 37, 42, 48, 53, 59, 64]  lr decay epochs : [43, 54]  learning rate : 0.0  momentum : 0.9  weight decay : 0.0005  lr warmup : None  backbone : resnet50  backbone path : None  num workers : 4  AMP : False  precision : fp32 
+Using seed = 4198
+loading annotations into memory...
+Traceback (most recent call last):
+  File "main.py", line 286, in <module>
+    train(train_loop_func, logger, args)
+  File "main.py", line 146, in train
+    cocoGt = get_coco_ground_truth(args)
+  File "/workspace/benchmark/Detection/SSD/ssd/data.py", line 73, in get_coco_ground_truth
+    cocoGt = COCO(annotation_file=val_annotate, use_ext=True)
+  File "/opt/conda/lib/python3.8/site-packages/pycocotools/coco.py", line 89, in __init__
+    dataset = json.load(open(annotation_file, 'r'))
+FileNotFoundError: [Errno 2] No such file or directory: '/data/object_detection/annotations/instances_val2017.json'
+ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 5329) of binary: /opt/conda/bin/python
+Traceback (most recent call last):
+  File "/opt/conda/bin/torchrun", line 33, in <module>
+    sys.exit(load_entry_point('torch==1.13.0a0+d0d6b1f', 'console_scripts', 'torchrun')())
+  File "/opt/conda/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper
+    return f(*args, **kwargs)
+  File "/opt/conda/lib/python3.8/site-packages/torch/distributed/run.py", line 762, in main
+    run(args)
+  File "/opt/conda/lib/python3.8/site-packages/torch/distributed/run.py", line 753, in run
+    elastic_launch(
+  File "/opt/conda/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 132, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/opt/conda/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent
+    raise ChildFailedError(
+torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
+============================================================
+main.py FAILED
+------------------------------------------------------------
+Failures:
+  <NO_OTHER_FAILURES>
+------------------------------------------------------------
+Root Cause (first observed failure):
+[0]:
+  time      : 2023-07-12_20:15:40
+  host      : 24a2da7181c1
+  rank      : 0 (local_rank: 0)
+  exitcode  : 1 (pid: 5329)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+============================================================
+DONE!
diff --git a/pytorch/results/4090laptop_v1/PyTorch_SSD_FP32/benchmark.para b/pytorch/results/4090laptop_v1/PyTorch_SSD_FP32/benchmark.para
new file mode 100755
index 00000000..e35db812
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_SSD_FP32/benchmark.para
@@ -0,0 +1,2 @@
+GLOBAL_BATCH 48
+GPU 1
diff --git a/pytorch/results/4090laptop_v1/PyTorch_bert_base_squad_FP16/12-07-2023_20-20-52.txt b/pytorch/results/4090laptop_v1/PyTorch_bert_base_squad_FP16/12-07-2023_20-20-52.txt
new file mode 100755
index 00000000..f0a883c3
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_bert_base_squad_FP16/12-07-2023_20-20-52.txt
@@ -0,0 +1,48 @@
+Container nvidia build =  46164382
+out dir is .
+fp16 activated!
+torchrun --nproc_per_node=1 run_squad.py --init_checkpoint=/data/bert_base/bert_base_uncased.pt --do_train --train_file=/data/squad/v1.1/train-v1.1.json --train_batch_size=48  --do_lower_case  --bert_model=bert-large-uncased  --learning_rate=0.0  --warmup_proportion=0.1 --seed=1  --num_train_epochs=2.0  --max_seq_length=384  --doc_stride=128  --output_dir=.  --vocab_file=/data/bert_base/bert-base-uncased-vocab.txt  --config_file=/data/bert_base/bert_config.json  --max_steps=100   --fp16  |& tee ./logfile.txt
+07/12/2023 20:20:55 - INFO - torch.distributed.distributed_c10d -   Added key: store_based_barrier_key:1 to store for rank: 0
+07/12/2023 20:20:55 - INFO - torch.distributed.distributed_c10d -   Rank 0: Completed store-based barrier for key:store_based_barrier_key:1 with 1 nodes.
+device: cuda:0 n_gpu: 1, distributed training: True, 16-bits training: True
+DLL 2023-07-12 20:20:55.811283 - PARAMETER Config : ["Namespace(amp=False, bert_model='bert-large-uncased', cache_dir=None, config_file='/data/bert_base/bert_config.json', disable_progress_bar=False, do_eval=False, do_lower_case=True, do_predict=False, do_train=True, doc_stride=128, eval_script='evaluate.py', fp16=True, gradient_accumulation_steps=1, init_checkpoint='/data/bert_base/bert_base_uncased.pt', json_summary='results/dllogger.json', learning_rate=0.0, local_rank=0, log_freq=50, loss_scale=0, max_answer_length=30, max_query_length=64, max_seq_length=384, max_steps=100.0, n_best_size=20, no_cuda=False, null_score_diff_threshold=0.0, num_train_epochs=2.0, output_dir='.', predict_batch_size=8, predict_file=None, seed=1, skip_cache=False, skip_checkpoint=False, train_batch_size=48, train_file='/data/squad/v1.1/train-v1.1.json', use_env=False, verbose_logging=False, version_2_with_negative=False, vocab_file='/data/bert_base/bert-base-uncased-vocab.txt', warmup_proportion=0.1)"] 
+DLL 2023-07-12 20:20:55.811401 - PARAMETER SEED : 1 
+WARNING: Output directory . already exists and is not empty. ['tokenization_utils.py', 'requirements.txt', 'checkpoints', 'bert_configs', 'lamb_amp_opt', 'configurations.yml', 'schedulers.py', 'README.md', 'distillation', 'create_pretraining_data.py', '.gitmodules', 'data', 'vocab', 'tokenization.py', 'Dockerfile', 'extract_features.py', 'run.sub', 'images', 'processors', 'run_swag.py', 'run_pretraining.py', 'inference.py', 'config.json', 'scripts', 'modeling.py', 'LICENSE', 'logfile.txt', 'optimization.py', 'pytorch_model.bin', '.dockerignore', 'run_glue.py', 'run_squad.py', 'triton', 'NOTICE', '__pycache__', 'results', '.gitignore', 'utils.py', 'bind.sh', 'bind_pyt.py', 'file_utils.py']
+DLL 2023-07-12 20:21:00.032482 - PARAMETER loading_checkpoint : True 
+DLL 2023-07-12 20:21:00.032556 - PARAMETER loaded_checkpoint : True 
+DLL 2023-07-12 20:21:00.154267 - PARAMETER model_weights_num : 109488386 
+Selected optimization level O2:  FP16 training with FP32 batchnorm and FP32 master weights.
+
+Defaults for this optimization level are:
+enabled                : True
+opt_level              : O2
+cast_model_type        : torch.float16
+patch_torch_functions  : False
+keep_batchnorm_fp32    : True
+master_weights         : True
+loss_scale             : dynamic
+Processing user overrides (additional kwargs that are not None)...
+After processing overrides, optimization options are:
+enabled                : True
+opt_level              : O2
+cast_model_type        : torch.float16
+patch_torch_functions  : False
+keep_batchnorm_fp32    : False
+master_weights         : True
+loss_scale             : dynamic
+DLL 2023-07-12 20:21:04.025876 - PARAMETER train_start : True 
+DLL 2023-07-12 20:21:04.025955 - PARAMETER training_samples : 87599 
+DLL 2023-07-12 20:21:04.025971 - PARAMETER training_features : 88641 
+DLL 2023-07-12 20:21:04.025981 - PARAMETER train_batch_size : 48 
+DLL 2023-07-12 20:21:04.025991 - PARAMETER steps : 3648.0 
+
Iteration:   0%|          | 0/1847 [00:00<?, ?it/s]DLL 2023-07-12 20:21:05.521361 - Training Epoch: 0 Training Iteration: 1  step_loss : 5.870918273925781  learning_rate : 0.0 
+
Iteration:   0%|          | 1/1847 [00:00<12:58,  2.37it/s]
Iteration:   0%|          | 2/1847 [00:00<11:29,  2.68it/s]
Iteration:   0%|          | 3/1847 [00:01<11:03,  2.78it/s]
Iteration:   0%|          | 4/1847 [00:01<10:54,  2.82it/s]
Iteration:   0%|          | 5/1847 [00:01<10:44,  2.86it/s]
Iteration:   0%|          | 6/1847 [00:02<10:40,  2.87it/s]
Iteration:   0%|          | 7/1847 [00:02<10:40,  2.87it/s]
Iteration:   0%|          | 8/1847 [00:02<10:37,  2.89it/s]
Iteration:   0%|          | 9/1847 [00:03<10:34,  2.90it/s]
Iteration:   1%|          | 10/1847 [00:03<10:36,  2.89it/s]
Iteration:   1%|          | 11/1847 [00:03<10:35,  2.89it/s]
Iteration:   1%|          | 12/1847 [00:04<10:33,  2.90it/s]
Iteration:   1%|          | 13/1847 [00:04<10:33,  2.90it/s]
Iteration:   1%|          | 14/1847 [00:04<10:33,  2.89it/s]
Iteration:   1%|          | 15/1847 [00:05<10:31,  2.90it/s]
Iteration:   1%|          | 16/1847 [00:05<10:30,  2.90it/s]
Iteration:   1%|          | 17/1847 [00:05<10:32,  2.89it/s]
Iteration:   1%|          | 18/1847 [00:06<10:31,  2.89it/s]
Iteration:   1%|          | 19/1847 [00:06<10:30,  2.90it/s]
Iteration:   1%|          | 20/1847 [00:06<10:31,  2.89it/s]
Iteration:   1%|          | 21/1847 [00:07<10:31,  2.89it/s]
Iteration:   1%|          | 22/1847 [00:07<10:30,  2.90it/s]
Iteration:   1%|          | 23/1847 [00:08<10:30,  2.89it/s]
Iteration:   1%|▏         | 24/1847 [00:08<10:30,  2.89it/s]
Iteration:   1%|▏         | 25/1847 [00:08<10:29,  2.90it/s]
Iteration:   1%|▏         | 26/1847 [00:09<10:28,  2.90it/s]
Iteration:   1%|▏         | 27/1847 [00:09<10:29,  2.89it/s]
Iteration:   2%|▏         | 28/1847 [00:09<10:29,  2.89it/s]
Iteration:   2%|▏         | 29/1847 [00:10<10:28,  2.89it/s]
Iteration:   2%|▏         | 30/1847 [00:10<10:28,  2.89it/s]
Iteration:   2%|▏         | 31/1847 [00:10<10:28,  2.89it/s]
Iteration:   2%|▏         | 32/1847 [00:11<10:28,  2.89it/s]
Iteration:   2%|▏         | 33/1847 [00:11<10:27,  2.89it/s]
Iteration:   2%|▏         | 34/1847 [00:11<10:27,  2.89it/s]
Iteration:   2%|▏         | 35/1847 [00:12<10:27,  2.89it/s]
Iteration:   2%|▏         | 36/1847 [00:12<10:26,  2.89it/s]
Iteration:   2%|▏         | 37/1847 [00:12<10:26,  2.89it/s]
Iteration:   2%|▏         | 38/1847 [00:13<10:25,  2.89it/s]
Iteration:   2%|▏         | 39/1847 [00:13<10:25,  2.89it/s]
Iteration:   2%|▏         | 40/1847 [00:13<10:25,  2.89it/s]
Iteration:   2%|▏         | 41/1847 [00:14<10:24,  2.89it/s]
Iteration:   2%|▏         | 42/1847 [00:14<10:24,  2.89it/s]
Iteration:   2%|▏         | 43/1847 [00:14<10:24,  2.89it/s]
Iteration:   2%|▏         | 44/1847 [00:15<10:24,  2.89it/s]
Iteration:   2%|▏         | 45/1847 [00:15<10:22,  2.89it/s]
Iteration:   2%|▏         | 46/1847 [00:15<10:22,  2.89it/s]
Iteration:   3%|▎         | 47/1847 [00:16<10:23,  2.89it/s]
Iteration:   3%|▎         | 48/1847 [00:16<10:22,  2.89it/s]
Iteration:   3%|▎         | 49/1847 [00:17<10:21,  2.89it/s]
Iteration:   3%|▎         | 50/1847 [00:17<10:23,  2.88it/s]DLL 2023-07-12 20:21:22.800333 - Training Epoch: 0 Training Iteration: 51  step_loss : 5.835312843322754  learning_rate : 0.0 
+
Iteration:   3%|▎         | 51/1847 [00:17<10:23,  2.88it/s]
Iteration:   3%|▎         | 52/1847 [00:18<10:22,  2.88it/s]
Iteration:   3%|▎         | 53/1847 [00:18<10:21,  2.88it/s]
Iteration:   3%|▎         | 54/1847 [00:18<10:23,  2.88it/s]
Iteration:   3%|▎         | 55/1847 [00:19<10:22,  2.88it/s]
Iteration:   3%|▎         | 56/1847 [00:19<10:20,  2.89it/s]
Iteration:   3%|▎         | 57/1847 [00:19<10:20,  2.88it/s]
Iteration:   3%|▎         | 58/1847 [00:20<10:21,  2.88it/s]
Iteration:   3%|▎         | 59/1847 [00:20<10:19,  2.88it/s]
Iteration:   3%|▎         | 60/1847 [00:20<10:18,  2.89it/s]
Iteration:   3%|▎         | 61/1847 [00:21<10:19,  2.88it/s]
Iteration:   3%|▎         | 62/1847 [00:21<10:19,  2.88it/s]
Iteration:   3%|▎         | 63/1847 [00:21<10:19,  2.88it/s]
Iteration:   3%|▎         | 64/1847 [00:22<10:18,  2.88it/s]
Iteration:   4%|▎         | 65/1847 [00:22<10:18,  2.88it/s]
Iteration:   4%|▎         | 66/1847 [00:22<10:18,  2.88it/s]
Iteration:   4%|▎         | 67/1847 [00:23<10:17,  2.88it/s]
Iteration:   4%|▎         | 68/1847 [00:23<10:17,  2.88it/s]
Iteration:   4%|▎         | 69/1847 [00:23<10:17,  2.88it/s]
Iteration:   4%|▍         | 70/1847 [00:24<10:17,  2.88it/s]
Iteration:   4%|▍         | 71/1847 [00:24<10:16,  2.88it/s]
Iteration:   4%|▍         | 72/1847 [00:24<10:16,  2.88it/s]
Iteration:   4%|▍         | 73/1847 [00:25<10:17,  2.88it/s]
Iteration:   4%|▍         | 74/1847 [00:25<10:16,  2.88it/s]
Iteration:   4%|▍         | 75/1847 [00:26<10:15,  2.88it/s]
Iteration:   4%|▍         | 76/1847 [00:26<10:15,  2.88it/s]
Iteration:   4%|▍         | 77/1847 [00:26<10:16,  2.87it/s]
Iteration:   4%|▍         | 78/1847 [00:27<10:15,  2.87it/s]
Iteration:   4%|▍         | 79/1847 [00:27<10:14,  2.88it/s]
Iteration:   4%|▍         | 80/1847 [00:27<10:14,  2.88it/s]
Iteration:   4%|▍         | 81/1847 [00:28<10:14,  2.87it/s]
Iteration:   4%|▍         | 82/1847 [00:28<10:14,  2.87it/s]
Iteration:   4%|▍         | 83/1847 [00:28<10:12,  2.88it/s]
Iteration:   5%|▍         | 84/1847 [00:29<10:13,  2.88it/s]
Iteration:   5%|▍         | 85/1847 [00:29<10:13,  2.87it/s]
Iteration:   5%|▍         | 86/1847 [00:29<10:10,  2.88it/s]
Iteration:   5%|▍         | 87/1847 [00:30<10:11,  2.88it/s]
Iteration:   5%|▍         | 88/1847 [00:30<10:12,  2.87it/s]
Iteration:   5%|▍         | 89/1847 [00:30<10:11,  2.87it/s]
Iteration:   5%|▍         | 90/1847 [00:31<10:10,  2.88it/s]
Iteration:   5%|▍         | 91/1847 [00:31<10:10,  2.87it/s]
Iteration:   5%|▍         | 92/1847 [00:31<10:11,  2.87it/s]
Iteration:   5%|▌         | 93/1847 [00:32<10:10,  2.87it/s]
Iteration:   5%|▌         | 94/1847 [00:32<10:09,  2.88it/s]
Iteration:   5%|▌         | 95/1847 [00:32<10:09,  2.88it/s]
Iteration:   5%|▌         | 96/1847 [00:33<10:09,  2.87it/s]
Iteration:   5%|▌         | 97/1847 [00:33<10:10,  2.87it/s]
Iteration:   5%|▌         | 98/1847 [00:34<10:08,  2.87it/s]
Iteration:   5%|▌         | 99/1847 [00:34<10:07,  2.88it/s]
Iteration:   5%|▌         | 100/1847 [00:34<10:08,  2.87it/s]DLL 2023-07-12 20:21:40.177844 - Training Epoch: 0 Training Iteration: 101  step_loss : 5.846651554107666  learning_rate : 0.0 
+
Iteration:   5%|▌         | 101/1847 [00:35<10:08,  2.87it/s]
Iteration:   5%|▌         | 101/1847 [00:35<10:06,  2.88it/s]
+
Iteration:   0%|          | 0/1847 [00:00<?, ?it/s]
Iteration:   0%|          | 0/1847 [00:00<?, ?it/s]
+DLL 2023-07-12 20:21:40.700208 -  e2e_train_time : 35.08303356170654  training_sequences_per_second : 136.81827118961698  final_loss : 5.846651554107666 
+
+real	0m50.822s
+user	0m46.192s
+sys	0m4.905s
+DONE!
diff --git a/pytorch/results/4090laptop_v1/PyTorch_bert_base_squad_FP16/benchmark.para b/pytorch/results/4090laptop_v1/PyTorch_bert_base_squad_FP16/benchmark.para
new file mode 100755
index 00000000..e35db812
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_bert_base_squad_FP16/benchmark.para
@@ -0,0 +1,2 @@
+GLOBAL_BATCH 48
+GPU 1
diff --git a/pytorch/results/4090laptop_v1/PyTorch_bert_base_squad_FP32/12-07-2023_20-09-35.txt b/pytorch/results/4090laptop_v1/PyTorch_bert_base_squad_FP32/12-07-2023_20-09-35.txt
new file mode 100755
index 00000000..3759b4b8
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_bert_base_squad_FP32/12-07-2023_20-09-35.txt
@@ -0,0 +1,33 @@
+Container nvidia build =  46164382
+out dir is .
+torchrun --nproc_per_node=1 run_squad.py --init_checkpoint=/data/bert_base/bert_base_uncased.pt --do_train --train_file=/data/squad/v1.1/train-v1.1.json --train_batch_size=24  --do_lower_case  --bert_model=bert-large-uncased  --learning_rate=0.0  --warmup_proportion=0.1 --seed=1  --num_train_epochs=2.0  --max_seq_length=384  --doc_stride=128  --output_dir=.  --vocab_file=/data/bert_base/bert-base-uncased-vocab.txt  --config_file=/data/bert_base/bert_config.json  --max_steps=100   |& tee ./logfile.txt
+07/12/2023 20:09:38 - INFO - torch.distributed.distributed_c10d -   Added key: store_based_barrier_key:1 to store for rank: 0
+07/12/2023 20:09:38 - INFO - torch.distributed.distributed_c10d -   Rank 0: Completed store-based barrier for key:store_based_barrier_key:1 with 1 nodes.
+device: cuda:0 n_gpu: 1, distributed training: True, 16-bits training: False
+DLL 2023-07-12 20:09:38.211467 - PARAMETER Config : ["Namespace(amp=False, bert_model='bert-large-uncased', cache_dir=None, config_file='/data/bert_base/bert_config.json', disable_progress_bar=False, do_eval=False, do_lower_case=True, do_predict=False, do_train=True, doc_stride=128, eval_script='evaluate.py', fp16=False, gradient_accumulation_steps=1, init_checkpoint='/data/bert_base/bert_base_uncased.pt', json_summary='results/dllogger.json', learning_rate=0.0, local_rank=0, log_freq=50, loss_scale=0, max_answer_length=30, max_query_length=64, max_seq_length=384, max_steps=100.0, n_best_size=20, no_cuda=False, null_score_diff_threshold=0.0, num_train_epochs=2.0, output_dir='.', predict_batch_size=8, predict_file=None, seed=1, skip_cache=False, skip_checkpoint=False, train_batch_size=24, train_file='/data/squad/v1.1/train-v1.1.json', use_env=False, verbose_logging=False, version_2_with_negative=False, vocab_file='/data/bert_base/bert-base-uncased-vocab.txt', warmup_proportion=0.1)"] 
+DLL 2023-07-12 20:09:38.211565 - PARAMETER SEED : 1 
+WARNING: Output directory . already exists and is not empty. ['tokenization_utils.py', 'requirements.txt', 'checkpoints', 'bert_configs', 'lamb_amp_opt', 'configurations.yml', 'schedulers.py', 'README.md', 'distillation', 'create_pretraining_data.py', '.gitmodules', 'data', 'vocab', 'tokenization.py', 'Dockerfile', 'extract_features.py', 'run.sub', 'images', 'processors', 'run_swag.py', 'run_pretraining.py', 'inference.py', 'config.json', 'scripts', 'modeling.py', 'LICENSE', 'logfile.txt', 'optimization.py', 'pytorch_model.bin', '.dockerignore', 'run_glue.py', 'run_squad.py', 'triton', 'NOTICE', '__pycache__', 'results', '.gitignore', 'utils.py', 'bind.sh', 'bind_pyt.py', 'file_utils.py']
+DLL 2023-07-12 20:09:42.371454 - PARAMETER loading_checkpoint : True 
+DLL 2023-07-12 20:09:42.371529 - PARAMETER loaded_checkpoint : True 
+DLL 2023-07-12 20:09:42.494884 - PARAMETER model_weights_num : 109488386 
+DLL 2023-07-12 20:09:46.289700 - PARAMETER train_start : True 
+DLL 2023-07-12 20:09:46.289765 - PARAMETER training_samples : 87599 
+DLL 2023-07-12 20:09:46.289777 - PARAMETER training_features : 88641 
+DLL 2023-07-12 20:09:46.289787 - PARAMETER train_batch_size : 24 
+DLL 2023-07-12 20:09:46.289794 - PARAMETER steps : 7298.0 
+
Iteration:   0%|          | 0/3694 [00:00<?, ?it/s]/workspace/benchmark/LanguageModeling/BERT/optimization.py:150: UserWarning: This overload of add_ is deprecated:
+	add_(Number alpha, Tensor other)
+Consider using one of the following signatures instead:
+	add_(Tensor other, *, Number alpha) (Triggered internally at /opt/pytorch/pytorch/torch/csrc/utils/python_arg_parser.cpp:1405.)
+  next_m.mul_(beta1).add_(1 - beta1, grad)
+DLL 2023-07-12 20:09:47.743170 - Training Epoch: 0 Training Iteration: 1  step_loss : 5.905476093292236  learning_rate : 0.0 
+
Iteration:   0%|          | 1/3694 [00:00<25:59,  2.37it/s]
Iteration:   0%|          | 2/3694 [00:00<23:08,  2.66it/s]
Iteration:   0%|          | 3/3694 [00:01<22:22,  2.75it/s]
Iteration:   0%|          | 4/3694 [00:01<22:07,  2.78it/s]
Iteration:   0%|          | 5/3694 [00:01<21:47,  2.82it/s]
Iteration:   0%|          | 6/3694 [00:02<21:39,  2.84it/s]
Iteration:   0%|          | 7/3694 [00:02<21:40,  2.83it/s]
Iteration:   0%|          | 8/3694 [00:02<21:33,  2.85it/s]
Iteration:   0%|          | 9/3694 [00:03<21:27,  2.86it/s]
Iteration:   0%|          | 10/3694 [00:03<21:30,  2.86it/s]
Iteration:   0%|          | 11/3694 [00:03<21:30,  2.85it/s]
Iteration:   0%|          | 12/3694 [00:04<21:26,  2.86it/s]
Iteration:   0%|          | 13/3694 [00:04<21:25,  2.86it/s]
Iteration:   0%|          | 14/3694 [00:04<21:29,  2.85it/s]
Iteration:   0%|          | 15/3694 [00:05<21:27,  2.86it/s]
Iteration:   0%|          | 16/3694 [00:05<21:24,  2.86it/s]
Iteration:   0%|          | 17/3694 [00:06<21:24,  2.86it/s]
Iteration:   0%|          | 18/3694 [00:06<21:26,  2.86it/s]
Iteration:   1%|          | 19/3694 [00:06<21:27,  2.85it/s]
Iteration:   1%|          | 20/3694 [00:07<21:24,  2.86it/s]
Iteration:   1%|          | 21/3694 [00:07<21:24,  2.86it/s]
Iteration:   1%|          | 22/3694 [00:07<21:25,  2.86it/s]
Iteration:   1%|          | 23/3694 [00:08<21:26,  2.85it/s]
Iteration:   1%|          | 24/3694 [00:08<21:23,  2.86it/s]
Iteration:   1%|          | 25/3694 [00:08<21:23,  2.86it/s]
Iteration:   1%|          | 26/3694 [00:09<21:26,  2.85it/s]
Iteration:   1%|          | 27/3694 [00:09<21:26,  2.85it/s]
Iteration:   1%|          | 28/3694 [00:09<21:24,  2.85it/s]
Iteration:   1%|          | 29/3694 [00:10<21:23,  2.86it/s]
Iteration:   1%|          | 30/3694 [00:10<21:24,  2.85it/s]
Iteration:   1%|          | 31/3694 [00:10<21:25,  2.85it/s]
Iteration:   1%|          | 32/3694 [00:11<21:24,  2.85it/s]
Iteration:   1%|          | 33/3694 [00:11<21:23,  2.85it/s]
Iteration:   1%|          | 34/3694 [00:11<21:25,  2.85it/s]
Iteration:   1%|          | 35/3694 [00:12<21:26,  2.85it/s]
Iteration:   1%|          | 36/3694 [00:12<21:23,  2.85it/s]
Iteration:   1%|          | 37/3694 [00:13<21:23,  2.85it/s]
Iteration:   1%|          | 38/3694 [00:13<21:23,  2.85it/s]
Iteration:   1%|          | 39/3694 [00:13<21:25,  2.84it/s]
Iteration:   1%|          | 40/3694 [00:14<21:24,  2.85it/s]
Iteration:   1%|          | 41/3694 [00:14<21:21,  2.85it/s]
Iteration:   1%|          | 42/3694 [00:14<21:21,  2.85it/s]
Iteration:   1%|          | 43/3694 [00:15<21:23,  2.84it/s]
Iteration:   1%|          | 44/3694 [00:15<21:23,  2.84it/s]
Iteration:   1%|          | 45/3694 [00:15<21:20,  2.85it/s]
Iteration:   1%|          | 46/3694 [00:16<21:20,  2.85it/s]
Iteration:   1%|▏         | 47/3694 [00:16<21:22,  2.84it/s]
Iteration:   1%|▏         | 48/3694 [00:16<21:22,  2.84it/s]
Iteration:   1%|▏         | 49/3694 [00:17<21:19,  2.85it/s]
Iteration:   1%|▏         | 50/3694 [00:17<21:19,  2.85it/s]DLL 2023-07-12 20:10:05.261178 - Training Epoch: 0 Training Iteration: 51  step_loss : 5.92388916015625  learning_rate : 0.0 
+
Iteration:   1%|▏         | 51/3694 [00:17<21:21,  2.84it/s]
Iteration:   1%|▏         | 52/3694 [00:18<21:21,  2.84it/s]
Iteration:   1%|▏         | 53/3694 [00:18<21:19,  2.84it/s]
Iteration:   1%|▏         | 54/3694 [00:18<21:19,  2.84it/s]
Iteration:   1%|▏         | 55/3694 [00:19<21:21,  2.84it/s]
Iteration:   2%|▏         | 56/3694 [00:19<21:22,  2.84it/s]
Iteration:   2%|▏         | 57/3694 [00:20<21:19,  2.84it/s]
Iteration:   2%|▏         | 58/3694 [00:20<21:17,  2.85it/s]
Iteration:   2%|▏         | 59/3694 [00:20<21:20,  2.84it/s]
Iteration:   2%|▏         | 60/3694 [00:21<21:20,  2.84it/s]
Iteration:   2%|▏         | 61/3694 [00:21<21:18,  2.84it/s]
Iteration:   2%|▏         | 62/3694 [00:21<21:18,  2.84it/s]
Iteration:   2%|▏         | 63/3694 [00:22<21:18,  2.84it/s]
Iteration:   2%|▏         | 64/3694 [00:22<21:18,  2.84it/s]
Iteration:   2%|▏         | 65/3694 [00:22<21:17,  2.84it/s]
Iteration:   2%|▏         | 66/3694 [00:23<21:16,  2.84it/s]
Iteration:   2%|▏         | 67/3694 [00:23<21:17,  2.84it/s]
Iteration:   2%|▏         | 68/3694 [00:23<21:19,  2.83it/s]
Iteration:   2%|▏         | 69/3694 [00:24<21:20,  2.83it/s]
Iteration:   2%|▏         | 70/3694 [00:24<21:15,  2.84it/s]
Iteration:   2%|▏         | 71/3694 [00:24<21:15,  2.84it/s]
Iteration:   2%|▏         | 72/3694 [00:25<21:16,  2.84it/s]
Iteration:   2%|▏         | 73/3694 [00:25<21:16,  2.84it/s]
Iteration:   2%|▏         | 74/3694 [00:26<21:13,  2.84it/s]
Iteration:   2%|▏         | 75/3694 [00:26<21:13,  2.84it/s]
Iteration:   2%|▏         | 76/3694 [00:26<21:15,  2.84it/s]
Iteration:   2%|▏         | 77/3694 [00:27<21:17,  2.83it/s]
Iteration:   2%|▏         | 78/3694 [00:27<21:13,  2.84it/s]
Iteration:   2%|▏         | 79/3694 [00:27<21:14,  2.84it/s]
Iteration:   2%|▏         | 80/3694 [00:28<21:17,  2.83it/s]
Iteration:   2%|▏         | 81/3694 [00:28<21:15,  2.83it/s]
Iteration:   2%|▏         | 82/3694 [00:28<21:13,  2.84it/s]
Iteration:   2%|▏         | 83/3694 [00:29<21:14,  2.83it/s]
Iteration:   2%|▏         | 84/3694 [00:29<21:13,  2.83it/s]
Iteration:   2%|▏         | 85/3694 [00:29<21:15,  2.83it/s]
Iteration:   2%|▏         | 86/3694 [00:30<21:16,  2.83it/s]
Iteration:   2%|▏         | 87/3694 [00:30<21:11,  2.84it/s]
Iteration:   2%|▏         | 88/3694 [00:30<21:11,  2.84it/s]
Iteration:   2%|▏         | 89/3694 [00:31<21:13,  2.83it/s]
Iteration:   2%|▏         | 90/3694 [00:31<21:12,  2.83it/s]
Iteration:   2%|▏         | 91/3694 [00:32<21:09,  2.84it/s]
Iteration:   2%|▏         | 92/3694 [00:32<21:08,  2.84it/s]
Iteration:   3%|▎         | 93/3694 [00:32<21:09,  2.84it/s]
Iteration:   3%|▎         | 94/3694 [00:33<21:12,  2.83it/s]
Iteration:   3%|▎         | 95/3694 [00:33<21:10,  2.83it/s]
Iteration:   3%|▎         | 96/3694 [00:33<21:09,  2.83it/s]
Iteration:   3%|▎         | 97/3694 [00:34<21:10,  2.83it/s]
Iteration:   3%|▎         | 98/3694 [00:34<21:09,  2.83it/s]
Iteration:   3%|▎         | 99/3694 [00:34<21:10,  2.83it/s]
Iteration:   3%|▎         | 100/3694 [00:35<21:08,  2.83it/s]DLL 2023-07-12 20:10:22.889770 - Training Epoch: 0 Training Iteration: 101  step_loss : 5.886537551879883  learning_rate : 0.0 
+
Iteration:   3%|▎         | 101/3694 [00:35<21:07,  2.84it/s]
Iteration:   3%|▎         | 101/3694 [00:35<21:05,  2.84it/s]
+
Iteration:   0%|          | 0/3694 [00:00<?, ?it/s]
Iteration:   0%|          | 0/3694 [00:00<?, ?it/s]
+DLL 2023-07-12 20:10:23.442101 -  e2e_train_time : 35.57346034049988  training_sequences_per_second : 67.46602599319343  final_loss : 5.886537551879883 
+
+real	0m50.812s
+user	0m44.221s
+sys	0m7.034s
+DONE!
diff --git a/pytorch/results/4090laptop_v1/PyTorch_bert_base_squad_FP32/benchmark.para b/pytorch/results/4090laptop_v1/PyTorch_bert_base_squad_FP32/benchmark.para
new file mode 100755
index 00000000..33536c8a
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_bert_base_squad_FP32/benchmark.para
@@ -0,0 +1,2 @@
+GLOBAL_BATCH 24
+GPU 1
diff --git a/pytorch/results/4090laptop_v1/PyTorch_bert_large_squad_FP16/12-07-2023_20-07-03.txt b/pytorch/results/4090laptop_v1/PyTorch_bert_large_squad_FP16/12-07-2023_20-07-03.txt
new file mode 100755
index 00000000..40e4f8c1
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_bert_large_squad_FP16/12-07-2023_20-07-03.txt
@@ -0,0 +1,48 @@
+Container nvidia build =  46164382
+out dir is .
+fp16 activated!
+torchrun --nproc_per_node=1 run_squad.py --init_checkpoint=/data/bert_large/bert_large_uncased.pt --do_train --train_file=/data/squad/v1.1/train-v1.1.json --train_batch_size=16  --do_lower_case  --bert_model=bert-large-uncased  --learning_rate=0.0  --warmup_proportion=0.1 --seed=1  --num_train_epochs=2.0  --max_seq_length=384  --doc_stride=128  --output_dir=.  --vocab_file=/data/bert_large/bert-large-uncased-vocab.txt  --config_file=/data/bert_large/bert_config.json  --max_steps=100   --fp16  |& tee ./logfile.txt
+07/12/2023 20:07:06 - INFO - torch.distributed.distributed_c10d -   Added key: store_based_barrier_key:1 to store for rank: 0
+07/12/2023 20:07:06 - INFO - torch.distributed.distributed_c10d -   Rank 0: Completed store-based barrier for key:store_based_barrier_key:1 with 1 nodes.
+device: cuda:0 n_gpu: 1, distributed training: True, 16-bits training: True
+DLL 2023-07-12 20:07:06.498364 - PARAMETER Config : ["Namespace(amp=False, bert_model='bert-large-uncased', cache_dir=None, config_file='/data/bert_large/bert_config.json', disable_progress_bar=False, do_eval=False, do_lower_case=True, do_predict=False, do_train=True, doc_stride=128, eval_script='evaluate.py', fp16=True, gradient_accumulation_steps=1, init_checkpoint='/data/bert_large/bert_large_uncased.pt', json_summary='results/dllogger.json', learning_rate=0.0, local_rank=0, log_freq=50, loss_scale=0, max_answer_length=30, max_query_length=64, max_seq_length=384, max_steps=100.0, n_best_size=20, no_cuda=False, null_score_diff_threshold=0.0, num_train_epochs=2.0, output_dir='.', predict_batch_size=8, predict_file=None, seed=1, skip_cache=False, skip_checkpoint=False, train_batch_size=16, train_file='/data/squad/v1.1/train-v1.1.json', use_env=False, verbose_logging=False, version_2_with_negative=False, vocab_file='/data/bert_large/bert-large-uncased-vocab.txt', warmup_proportion=0.1)"] 
+DLL 2023-07-12 20:07:06.498453 - PARAMETER SEED : 1 
+WARNING: Output directory . already exists and is not empty. ['tokenization_utils.py', 'requirements.txt', 'checkpoints', 'bert_configs', 'lamb_amp_opt', 'configurations.yml', 'schedulers.py', 'README.md', 'distillation', 'create_pretraining_data.py', '.gitmodules', 'data', 'vocab', 'tokenization.py', 'Dockerfile', 'extract_features.py', 'run.sub', 'images', 'processors', 'run_swag.py', 'run_pretraining.py', 'inference.py', 'config.json', 'scripts', 'modeling.py', 'LICENSE', 'logfile.txt', 'optimization.py', 'pytorch_model.bin', '.dockerignore', 'run_glue.py', 'run_squad.py', 'triton', 'NOTICE', '__pycache__', 'results', '.gitignore', 'utils.py', 'bind.sh', 'bind_pyt.py', 'file_utils.py']
+DLL 2023-07-12 20:07:13.965754 - PARAMETER loading_checkpoint : True 
+DLL 2023-07-12 20:07:13.965833 - PARAMETER loaded_checkpoint : True 
+DLL 2023-07-12 20:07:14.146662 - PARAMETER model_weights_num : 335150082 
+Selected optimization level O2:  FP16 training with FP32 batchnorm and FP32 master weights.
+
+Defaults for this optimization level are:
+enabled                : True
+opt_level              : O2
+cast_model_type        : torch.float16
+patch_torch_functions  : False
+keep_batchnorm_fp32    : True
+master_weights         : True
+loss_scale             : dynamic
+Processing user overrides (additional kwargs that are not None)...
+After processing overrides, optimization options are:
+enabled                : True
+opt_level              : O2
+cast_model_type        : torch.float16
+patch_torch_functions  : False
+keep_batchnorm_fp32    : False
+master_weights         : True
+loss_scale             : dynamic
+DLL 2023-07-12 20:07:18.016377 - PARAMETER train_start : True 
+DLL 2023-07-12 20:07:18.016440 - PARAMETER training_samples : 87599 
+DLL 2023-07-12 20:07:18.016453 - PARAMETER training_features : 88641 
+DLL 2023-07-12 20:07:18.016462 - PARAMETER train_batch_size : 16 
+DLL 2023-07-12 20:07:18.016470 - PARAMETER steps : 10948.0 
+
Iteration:   0%|          | 0/5541 [00:00<?, ?it/s]DLL 2023-07-12 20:07:19.546825 - Training Epoch: 0 Training Iteration: 1  step_loss : 6.130677223205566  learning_rate : 0.0 
+
Iteration:   0%|          | 1/5541 [00:00<43:59,  2.10it/s]
Iteration:   0%|          | 2/5541 [00:00<38:07,  2.42it/s]
Iteration:   0%|          | 3/5541 [00:01<36:41,  2.52it/s]
Iteration:   0%|          | 4/5541 [00:01<36:05,  2.56it/s]
Iteration:   0%|          | 5/5541 [00:01<35:27,  2.60it/s]
Iteration:   0%|          | 6/5541 [00:02<35:19,  2.61it/s]
Iteration:   0%|          | 7/5541 [00:02<35:17,  2.61it/s]
Iteration:   0%|          | 8/5541 [00:03<34:59,  2.64it/s]
Iteration:   0%|          | 9/5541 [00:03<34:58,  2.64it/s]
Iteration:   0%|          | 10/5541 [00:03<35:01,  2.63it/s]
Iteration:   0%|          | 11/5541 [00:04<34:50,  2.65it/s]
Iteration:   0%|          | 12/5541 [00:04<34:50,  2.65it/s]
Iteration:   0%|          | 13/5541 [00:05<34:55,  2.64it/s]
Iteration:   0%|          | 14/5541 [00:05<34:46,  2.65it/s]
Iteration:   0%|          | 15/5541 [00:05<34:51,  2.64it/s]
Iteration:   0%|          | 16/5541 [00:06<34:54,  2.64it/s]
Iteration:   0%|          | 17/5541 [00:06<34:47,  2.65it/s]
Iteration:   0%|          | 18/5541 [00:06<34:50,  2.64it/s]
Iteration:   0%|          | 19/5541 [00:07<34:53,  2.64it/s]
Iteration:   0%|          | 20/5541 [00:07<34:46,  2.65it/s]
Iteration:   0%|          | 21/5541 [00:08<34:52,  2.64it/s]
Iteration:   0%|          | 22/5541 [00:08<34:53,  2.64it/s]
Iteration:   0%|          | 23/5541 [00:08<34:46,  2.64it/s]
Iteration:   0%|          | 24/5541 [00:09<34:51,  2.64it/s]
Iteration:   0%|          | 25/5541 [00:09<34:50,  2.64it/s]
Iteration:   0%|          | 26/5541 [00:09<34:46,  2.64it/s]
Iteration:   0%|          | 27/5541 [00:10<34:51,  2.64it/s]
Iteration:   1%|          | 28/5541 [00:10<34:48,  2.64it/s]
Iteration:   1%|          | 29/5541 [00:11<34:44,  2.64it/s]
Iteration:   1%|          | 30/5541 [00:11<34:49,  2.64it/s]
Iteration:   1%|          | 31/5541 [00:11<34:48,  2.64it/s]
Iteration:   1%|          | 32/5541 [00:12<34:48,  2.64it/s]
Iteration:   1%|          | 33/5541 [00:12<34:49,  2.64it/s]
Iteration:   1%|          | 34/5541 [00:12<34:49,  2.64it/s]
Iteration:   1%|          | 35/5541 [00:13<34:48,  2.64it/s]
Iteration:   1%|          | 36/5541 [00:13<34:49,  2.63it/s]
Iteration:   1%|          | 37/5541 [00:14<34:47,  2.64it/s]
Iteration:   1%|          | 38/5541 [00:14<34:47,  2.64it/s]
Iteration:   1%|          | 39/5541 [00:14<34:48,  2.63it/s]
Iteration:   1%|          | 40/5541 [00:15<34:47,  2.64it/s]
Iteration:   1%|          | 41/5541 [00:15<34:48,  2.63it/s]
Iteration:   1%|          | 42/5541 [00:15<34:47,  2.63it/s]
Iteration:   1%|          | 43/5541 [00:16<34:47,  2.63it/s]
Iteration:   1%|          | 44/5541 [00:16<34:49,  2.63it/s]
Iteration:   1%|          | 45/5541 [00:17<34:48,  2.63it/s]
Iteration:   1%|          | 46/5541 [00:17<34:46,  2.63it/s]
Iteration:   1%|          | 47/5541 [00:17<34:47,  2.63it/s]
Iteration:   1%|          | 48/5541 [00:18<34:45,  2.63it/s]
Iteration:   1%|          | 49/5541 [00:18<34:46,  2.63it/s]
Iteration:   1%|          | 50/5541 [00:19<34:46,  2.63it/s]DLL 2023-07-12 20:07:38.488118 - Training Epoch: 0 Training Iteration: 51  step_loss : 6.042986869812012  learning_rate : 0.0 
+
Iteration:   1%|          | 51/5541 [00:19<34:46,  2.63it/s]
Iteration:   1%|          | 52/5541 [00:19<34:46,  2.63it/s]
Iteration:   1%|          | 53/5541 [00:20<34:46,  2.63it/s]
Iteration:   1%|          | 54/5541 [00:20<34:49,  2.63it/s]
Iteration:   1%|          | 55/5541 [00:20<34:49,  2.63it/s]
Iteration:   1%|          | 56/5541 [00:21<34:47,  2.63it/s]
Iteration:   1%|          | 57/5541 [00:21<34:46,  2.63it/s]
Iteration:   1%|          | 58/5541 [00:22<34:44,  2.63it/s]
Iteration:   1%|          | 59/5541 [00:22<34:43,  2.63it/s]
Iteration:   1%|          | 60/5541 [00:22<34:44,  2.63it/s]
Iteration:   1%|          | 61/5541 [00:23<34:44,  2.63it/s]
Iteration:   1%|          | 62/5541 [00:23<34:45,  2.63it/s]
Iteration:   1%|          | 63/5541 [00:23<34:41,  2.63it/s]
Iteration:   1%|          | 64/5541 [00:24<34:43,  2.63it/s]
Iteration:   1%|          | 65/5541 [00:24<34:45,  2.63it/s]
Iteration:   1%|          | 66/5541 [00:25<34:41,  2.63it/s]
Iteration:   1%|          | 67/5541 [00:25<34:41,  2.63it/s]
Iteration:   1%|          | 68/5541 [00:25<34:45,  2.62it/s]
Iteration:   1%|          | 69/5541 [00:26<34:43,  2.63it/s]
Iteration:   1%|▏         | 70/5541 [00:26<34:44,  2.63it/s]
Iteration:   1%|▏         | 71/5541 [00:27<34:43,  2.63it/s]
Iteration:   1%|▏         | 72/5541 [00:27<34:41,  2.63it/s]
Iteration:   1%|▏         | 73/5541 [00:27<34:42,  2.63it/s]
Iteration:   1%|▏         | 74/5541 [00:28<34:43,  2.62it/s]
Iteration:   1%|▏         | 75/5541 [00:28<34:42,  2.62it/s]
Iteration:   1%|▏         | 76/5541 [00:28<34:43,  2.62it/s]
Iteration:   1%|▏         | 77/5541 [00:29<34:42,  2.62it/s]
Iteration:   1%|▏         | 78/5541 [00:29<34:38,  2.63it/s]
Iteration:   1%|▏         | 79/5541 [00:30<34:39,  2.63it/s]
Iteration:   1%|▏         | 80/5541 [00:30<34:42,  2.62it/s]
Iteration:   1%|▏         | 81/5541 [00:30<34:41,  2.62it/s]
Iteration:   1%|▏         | 82/5541 [00:31<34:41,  2.62it/s]
Iteration:   1%|▏         | 83/5541 [00:31<34:44,  2.62it/s]
Iteration:   2%|▏         | 84/5541 [00:31<34:42,  2.62it/s]
Iteration:   2%|▏         | 85/5541 [00:32<34:42,  2.62it/s]
Iteration:   2%|▏         | 86/5541 [00:32<34:44,  2.62it/s]
Iteration:   2%|▏         | 87/5541 [00:33<34:40,  2.62it/s]
Iteration:   2%|▏         | 88/5541 [00:33<34:38,  2.62it/s]
Iteration:   2%|▏         | 89/5541 [00:33<34:39,  2.62it/s]
Iteration:   2%|▏         | 90/5541 [00:34<34:39,  2.62it/s]
Iteration:   2%|▏         | 91/5541 [00:34<34:39,  2.62it/s]
Iteration:   2%|▏         | 92/5541 [00:35<34:40,  2.62it/s]
Iteration:   2%|▏         | 93/5541 [00:35<34:38,  2.62it/s]
Iteration:   2%|▏         | 94/5541 [00:35<34:39,  2.62it/s]
Iteration:   2%|▏         | 95/5541 [00:36<34:40,  2.62it/s]
Iteration:   2%|▏         | 96/5541 [00:36<34:39,  2.62it/s]
Iteration:   2%|▏         | 97/5541 [00:36<34:38,  2.62it/s]
Iteration:   2%|▏         | 98/5541 [00:37<34:39,  2.62it/s]
Iteration:   2%|▏         | 99/5541 [00:37<34:36,  2.62it/s]
Iteration:   2%|▏         | 100/5541 [00:38<34:34,  2.62it/s]DLL 2023-07-12 20:07:57.543818 - Training Epoch: 0 Training Iteration: 101  step_loss : 5.969560623168945  learning_rate : 0.0 
+
Iteration:   2%|▏         | 101/5541 [00:38<34:35,  2.62it/s]
Iteration:   2%|▏         | 101/5541 [00:38<34:32,  2.63it/s]
+
Iteration:   0%|          | 0/5541 [00:00<?, ?it/s]
Iteration:   0%|          | 0/5541 [00:00<?, ?it/s]
+DLL 2023-07-12 20:07:58.895636 -  e2e_train_time : 38.480103731155396  training_sequences_per_second : 41.57992949235636  final_loss : 5.969560623168945 
+
+real	0m55.814s
+user	0m45.570s
+sys	0m12.740s
+DONE!
diff --git a/pytorch/results/4090laptop_v1/PyTorch_bert_large_squad_FP16/benchmark.para b/pytorch/results/4090laptop_v1/PyTorch_bert_large_squad_FP16/benchmark.para
new file mode 100755
index 00000000..75af4290
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_bert_large_squad_FP16/benchmark.para
@@ -0,0 +1,2 @@
+GLOBAL_BATCH 16
+GPU 1
diff --git a/pytorch/results/4090laptop_v1/PyTorch_bert_large_squad_FP32/12-07-2023_19-53-59.txt b/pytorch/results/4090laptop_v1/PyTorch_bert_large_squad_FP32/12-07-2023_19-53-59.txt
new file mode 100755
index 00000000..44c314fc
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_bert_large_squad_FP32/12-07-2023_19-53-59.txt
@@ -0,0 +1,34 @@
+Container nvidia build =  46164382
+out dir is .
+torchrun --nproc_per_node=1 run_squad.py --init_checkpoint=/data/bert_large/bert_large_uncased.pt --do_train --train_file=/data/squad/v1.1/train-v1.1.json --train_batch_size=8  --do_lower_case  --bert_model=bert-large-uncased  --learning_rate=0.0  --warmup_proportion=0.1 --seed=1  --num_train_epochs=2.0  --max_seq_length=384  --doc_stride=128  --output_dir=.  --vocab_file=/data/bert_large/bert-large-uncased-vocab.txt  --config_file=/data/bert_large/bert_config.json  --max_steps=100   |& tee ./logfile.txt
+07/12/2023 19:55:26 - INFO - torch.distributed.distributed_c10d -   Added key: store_based_barrier_key:1 to store for rank: 0
+07/12/2023 19:55:26 - INFO - torch.distributed.distributed_c10d -   Rank 0: Completed store-based barrier for key:store_based_barrier_key:1 with 1 nodes.
+device: cuda:0 n_gpu: 1, distributed training: True, 16-bits training: False
+DLL 2023-07-12 19:55:26.828243 - PARAMETER Config : ["Namespace(amp=False, bert_model='bert-large-uncased', cache_dir=None, config_file='/data/bert_large/bert_config.json', disable_progress_bar=False, do_eval=False, do_lower_case=True, do_predict=False, do_train=True, doc_stride=128, eval_script='evaluate.py', fp16=False, gradient_accumulation_steps=1, init_checkpoint='/data/bert_large/bert_large_uncased.pt', json_summary='results/dllogger.json', learning_rate=0.0, local_rank=0, log_freq=50, loss_scale=0, max_answer_length=30, max_query_length=64, max_seq_length=384, max_steps=100.0, n_best_size=20, no_cuda=False, null_score_diff_threshold=0.0, num_train_epochs=2.0, output_dir='.', predict_batch_size=8, predict_file=None, seed=1, skip_cache=False, skip_checkpoint=False, train_batch_size=8, train_file='/data/squad/v1.1/train-v1.1.json', use_env=False, verbose_logging=False, version_2_with_negative=False, vocab_file='/data/bert_large/bert-large-uncased-vocab.txt', warmup_proportion=0.1)"] 
+DLL 2023-07-12 19:55:26.828332 - PARAMETER SEED : 1 
+WARNING: Output directory . already exists and is not empty. ['tokenization_utils.py', 'requirements.txt', 'checkpoints', 'bert_configs', 'lamb_amp_opt', 'configurations.yml', 'schedulers.py', 'README.md', 'distillation', 'create_pretraining_data.py', '.gitmodules', 'data', 'vocab', 'tokenization.py', 'Dockerfile', 'extract_features.py', 'run.sub', 'images', 'processors', 'run_swag.py', 'run_pretraining.py', 'inference.py', 'scripts', 'modeling.py', 'LICENSE', 'logfile.txt', 'optimization.py', '.dockerignore', 'run_glue.py', 'run_squad.py', 'triton', 'NOTICE', '__pycache__', 'results', '.gitignore', 'utils.py', 'bind.sh', 'bind_pyt.py', 'file_utils.py']
+DLL 2023-07-12 19:55:33.563207 - PARAMETER loading_checkpoint : True 
+DLL 2023-07-12 19:55:33.563282 - PARAMETER loaded_checkpoint : True 
+DLL 2023-07-12 19:55:33.693000 - PARAMETER model_weights_num : 335150082 
+DLL 2023-07-12 19:56:55.687524 - PARAMETER Cached_train features_file : /data/squad/v1.1/train-v1.1.json_bert-large-uncased_384_128_64 
+DLL 2023-07-12 19:56:58.825143 - PARAMETER train_start : True 
+DLL 2023-07-12 19:56:58.825214 - PARAMETER training_samples : 87599 
+DLL 2023-07-12 19:56:58.825227 - PARAMETER training_features : 88641 
+DLL 2023-07-12 19:56:58.825236 - PARAMETER train_batch_size : 8 
+DLL 2023-07-12 19:56:58.825244 - PARAMETER steps : 21898.0 
+
Iteration:   0%|          | 0/11081 [00:00<?, ?it/s]/workspace/benchmark/LanguageModeling/BERT/optimization.py:150: UserWarning: This overload of add_ is deprecated:
+	add_(Number alpha, Tensor other)
+Consider using one of the following signatures instead:
+	add_(Tensor other, *, Number alpha) (Triggered internally at /opt/pytorch/pytorch/torch/csrc/utils/python_arg_parser.cpp:1405.)
+  next_m.mul_(beta1).add_(1 - beta1, grad)
+DLL 2023-07-12 19:57:00.312940 - Training Epoch: 0 Training Iteration: 1  step_loss : 6.100621223449707  learning_rate : 0.0 
+
Iteration:   0%|          | 1/11081 [00:00<1:35:42,  1.93it/s]
Iteration:   0%|          | 2/11081 [00:00<1:21:36,  2.26it/s]
Iteration:   0%|          | 3/11081 [00:01<1:17:18,  2.39it/s]
Iteration:   0%|          | 4/11081 [00:01<1:15:38,  2.44it/s]
Iteration:   0%|          | 5/11081 [00:02<1:14:09,  2.49it/s]
Iteration:   0%|          | 6/11081 [00:02<1:13:42,  2.50it/s]
Iteration:   0%|          | 7/11081 [00:02<1:13:24,  2.51it/s]
Iteration:   0%|          | 8/11081 [00:03<1:13:03,  2.53it/s]
Iteration:   0%|          | 9/11081 [00:03<1:13:03,  2.53it/s]
Iteration:   0%|          | 10/11081 [00:04<1:12:57,  2.53it/s]
Iteration:   0%|          | 11/11081 [00:04<1:12:47,  2.53it/s]
Iteration:   0%|          | 12/11081 [00:04<1:12:50,  2.53it/s]
Iteration:   0%|          | 13/11081 [00:05<1:12:51,  2.53it/s]
Iteration:   0%|          | 14/11081 [00:05<1:12:46,  2.53it/s]
Iteration:   0%|          | 15/11081 [00:06<1:12:50,  2.53it/s]
Iteration:   0%|          | 16/11081 [00:06<1:12:49,  2.53it/s]
Iteration:   0%|          | 17/11081 [00:06<1:12:48,  2.53it/s]
Iteration:   0%|          | 18/11081 [00:07<1:12:49,  2.53it/s]
Iteration:   0%|          | 19/11081 [00:07<1:12:51,  2.53it/s]
Iteration:   0%|          | 20/11081 [00:08<1:12:55,  2.53it/s]
Iteration:   0%|          | 21/11081 [00:08<1:12:58,  2.53it/s]
Iteration:   0%|          | 22/11081 [00:08<1:12:56,  2.53it/s]
Iteration:   0%|          | 23/11081 [00:09<1:13:01,  2.52it/s]
Iteration:   0%|          | 24/11081 [00:09<1:13:03,  2.52it/s]
Iteration:   0%|          | 25/11081 [00:09<1:13:04,  2.52it/s]
Iteration:   0%|          | 26/11081 [00:10<1:13:08,  2.52it/s]
Iteration:   0%|          | 27/11081 [00:10<1:13:08,  2.52it/s]
Iteration:   0%|          | 28/11081 [00:11<1:13:08,  2.52it/s]
Iteration:   0%|          | 29/11081 [00:11<1:13:10,  2.52it/s]
Iteration:   0%|          | 30/11081 [00:11<1:13:08,  2.52it/s]
Iteration:   0%|          | 31/11081 [00:12<1:13:11,  2.52it/s]
Iteration:   0%|          | 32/11081 [00:12<1:13:11,  2.52it/s]
Iteration:   0%|          | 33/11081 [00:13<1:13:06,  2.52it/s]
Iteration:   0%|          | 34/11081 [00:13<1:13:09,  2.52it/s]
Iteration:   0%|          | 35/11081 [00:13<1:13:04,  2.52it/s]
Iteration:   0%|          | 36/11081 [00:14<1:13:02,  2.52it/s]
Iteration:   0%|          | 37/11081 [00:14<1:13:07,  2.52it/s]
Iteration:   0%|          | 38/11081 [00:15<1:13:05,  2.52it/s]
Iteration:   0%|          | 39/11081 [00:15<1:13:00,  2.52it/s]
Iteration:   0%|          | 40/11081 [00:15<1:13:00,  2.52it/s]
Iteration:   0%|          | 41/11081 [00:16<1:12:57,  2.52it/s]
Iteration:   0%|          | 42/11081 [00:16<1:12:53,  2.52it/s]
Iteration:   0%|          | 43/11081 [00:17<1:12:44,  2.53it/s]
Iteration:   0%|          | 44/11081 [00:17<1:12:47,  2.53it/s]
Iteration:   0%|          | 45/11081 [00:17<1:12:46,  2.53it/s]
Iteration:   0%|          | 46/11081 [00:18<1:12:51,  2.52it/s]
Iteration:   0%|          | 47/11081 [00:18<1:12:57,  2.52it/s]
Iteration:   0%|          | 48/11081 [00:19<1:12:58,  2.52it/s]
Iteration:   0%|          | 49/11081 [00:19<1:12:59,  2.52it/s]
Iteration:   0%|          | 50/11081 [00:19<1:13:01,  2.52it/s]DLL 2023-07-12 19:57:20.098911 - Training Epoch: 0 Training Iteration: 51  step_loss : 6.082047462463379  learning_rate : 0.0 
+
Iteration:   0%|          | 51/11081 [00:20<1:13:02,  2.52it/s]
Iteration:   0%|          | 52/11081 [00:20<1:13:05,  2.51it/s]
Iteration:   0%|          | 53/11081 [00:21<1:13:09,  2.51it/s]
Iteration:   0%|          | 54/11081 [00:21<1:13:08,  2.51it/s]
Iteration:   0%|          | 55/11081 [00:21<1:13:06,  2.51it/s]
Iteration:   1%|          | 56/11081 [00:22<1:13:09,  2.51it/s]
Iteration:   1%|          | 57/11081 [00:22<1:13:09,  2.51it/s]
Iteration:   1%|          | 58/11081 [00:23<1:13:12,  2.51it/s]
Iteration:   1%|          | 59/11081 [00:23<1:13:11,  2.51it/s]
Iteration:   1%|          | 60/11081 [00:23<1:13:13,  2.51it/s]
Iteration:   1%|          | 61/11081 [00:24<1:13:12,  2.51it/s]
Iteration:   1%|          | 62/11081 [00:24<1:13:10,  2.51it/s]
Iteration:   1%|          | 63/11081 [00:25<1:13:14,  2.51it/s]
Iteration:   1%|          | 64/11081 [00:25<1:13:17,  2.51it/s]
Iteration:   1%|          | 65/11081 [00:25<1:13:14,  2.51it/s]
Iteration:   1%|          | 66/11081 [00:26<1:13:13,  2.51it/s]
Iteration:   1%|          | 67/11081 [00:26<1:13:11,  2.51it/s]
Iteration:   1%|          | 68/11081 [00:27<1:13:12,  2.51it/s]
Iteration:   1%|          | 69/11081 [00:27<1:13:14,  2.51it/s]
Iteration:   1%|          | 70/11081 [00:27<1:13:13,  2.51it/s]
Iteration:   1%|          | 71/11081 [00:28<1:13:13,  2.51it/s]
Iteration:   1%|          | 72/11081 [00:28<1:13:14,  2.51it/s]
Iteration:   1%|          | 73/11081 [00:29<1:13:13,  2.51it/s]
Iteration:   1%|          | 74/11081 [00:29<1:13:14,  2.50it/s]
Iteration:   1%|          | 75/11081 [00:29<1:13:17,  2.50it/s]
Iteration:   1%|          | 76/11081 [00:30<1:13:16,  2.50it/s]
Iteration:   1%|          | 77/11081 [00:30<1:13:15,  2.50it/s]
Iteration:   1%|          | 78/11081 [00:31<1:13:15,  2.50it/s]
Iteration:   1%|          | 79/11081 [00:31<1:13:21,  2.50it/s]
Iteration:   1%|          | 80/11081 [00:31<1:13:16,  2.50it/s]
Iteration:   1%|          | 81/11081 [00:32<1:13:12,  2.50it/s]
Iteration:   1%|          | 82/11081 [00:32<1:13:13,  2.50it/s]
Iteration:   1%|          | 83/11081 [00:33<1:13:17,  2.50it/s]
Iteration:   1%|          | 84/11081 [00:33<1:13:09,  2.51it/s]
Iteration:   1%|          | 85/11081 [00:33<1:13:14,  2.50it/s]
Iteration:   1%|          | 86/11081 [00:34<1:13:14,  2.50it/s]
Iteration:   1%|          | 87/11081 [00:34<1:13:10,  2.50it/s]
Iteration:   1%|          | 88/11081 [00:35<1:13:15,  2.50it/s]
Iteration:   1%|          | 89/11081 [00:35<1:13:19,  2.50it/s]
Iteration:   1%|          | 90/11081 [00:35<1:13:12,  2.50it/s]
Iteration:   1%|          | 91/11081 [00:36<1:13:14,  2.50it/s]
Iteration:   1%|          | 92/11081 [00:36<1:13:14,  2.50it/s]
Iteration:   1%|          | 93/11081 [00:37<1:13:15,  2.50it/s]
Iteration:   1%|          | 94/11081 [00:37<1:13:18,  2.50it/s]
Iteration:   1%|          | 95/11081 [00:37<1:13:14,  2.50it/s]
Iteration:   1%|          | 96/11081 [00:38<1:13:15,  2.50it/s]
Iteration:   1%|          | 97/11081 [00:38<1:13:13,  2.50it/s]
Iteration:   1%|          | 98/11081 [00:39<1:13:15,  2.50it/s]
Iteration:   1%|          | 99/11081 [00:39<1:13:19,  2.50it/s]
Iteration:   1%|          | 100/11081 [00:39<1:13:18,  2.50it/s]DLL 2023-07-12 19:57:40.070517 - Training Epoch: 0 Training Iteration: 101  step_loss : 6.0423126220703125  learning_rate : 0.0 
+
Iteration:   1%|          | 101/11081 [00:40<1:13:16,  2.50it/s]
Iteration:   1%|          | 101/11081 [00:40<1:12:58,  2.51it/s]
+
Iteration:   0%|          | 0/11081 [00:00<?, ?it/s]
Iteration:   0%|          | 0/11081 [00:00<?, ?it/s]
+DLL 2023-07-12 19:57:40.854985 -  e2e_train_time : 40.28011393547058  training_sequences_per_second : 19.86091700936133  final_loss : 6.0423126220703125 
+
+real	2m20.851s
+user	2m3.011s
+sys	0m18.196s
+DONE!
diff --git a/pytorch/results/4090laptop_v1/PyTorch_bert_large_squad_FP32/benchmark.para b/pytorch/results/4090laptop_v1/PyTorch_bert_large_squad_FP32/benchmark.para
new file mode 100755
index 00000000..6ea5ca92
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_bert_large_squad_FP32/benchmark.para
@@ -0,0 +1,2 @@
+GLOBAL_BATCH 8
+GPU 1
diff --git a/pytorch/results/4090laptop_v1/PyTorch_gnmt_FP16/12-07-2023_20-18-34.txt b/pytorch/results/4090laptop_v1/PyTorch_gnmt_FP16/12-07-2023_20-18-34.txt
new file mode 100755
index 00000000..f1ab13a8
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_gnmt_FP16/12-07-2023_20-18-34.txt
@@ -0,0 +1,233 @@
+0: thread affinity: {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}
+0: Collecting environment information...
+0: PyTorch version: 1.13.0a0+d0d6b1f
+Is debug build: False
+CUDA used to build PyTorch: 11.8
+ROCM used to build PyTorch: N/A
+
+OS: Ubuntu 20.04.5 LTS (x86_64)
+GCC version: (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0
+Clang version: Could not collect
+CMake version: version 3.22.2
+Libc version: glibc-2.31
+
+Python version: 3.8.13 | packaged by conda-forge | (default, Mar 25 2022, 06:04:10)  [GCC 10.3.0] (64-bit runtime)
+Python platform: Linux-5.19.0-46-generic-x86_64-with-glibc2.10
+Is CUDA available: True
+CUDA runtime version: 11.8.89
+GPU models and configuration: GPU 0: NVIDIA GeForce RTX 4090 Laptop GPU
+Nvidia driver version: 525.116.04
+cuDNN version: Probably one of the following:
+/usr/lib/x86_64-linux-gnu/libcudnn.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.6.0
+HIP runtime version: N/A
+MIOpen runtime version: N/A
+Is XNNPACK available: True
+
+Versions of relevant libraries:
+[pip3] functorch==0.3.0a0
+[pip3] numpy==1.22.2
+[pip3] pytorch-quantization==2.1.2
+[pip3] pytorch-transformers==1.1.0
+[pip3] torch==1.13.0a0+d0d6b1f
+[pip3] torch-tensorrt==1.3.0a0
+[pip3] torchtext==0.11.0a0
+[pip3] torchvision==0.14.0a0
+[conda] functorch                 0.3.0a0                  pypi_0    pypi
+[conda] mkl                       2020.4             h726a3e6_304    conda-forge
+[conda] mkl-include               2020.4             h726a3e6_304    conda-forge
+[conda] numpy                     1.22.2           py38h6ae9a64_0    conda-forge
+[conda] pytorch-quantization      2.1.2                    pypi_0    pypi
+[conda] pytorch-transformers      1.1.0                    pypi_0    pypi
+[conda] torch                     1.13.0a0+d0d6b1f          pypi_0    pypi
+[conda] torch-tensorrt            1.3.0a0                  pypi_0    pypi
+[conda] torchtext                 0.11.0a0                 pypi_0    pypi
+[conda] torchvision               0.14.0a0                 pypi_0    pypi
+0: Saving results to: gnmt
+0: Run arguments: Namespace(affinity='socket_unique_interleaved', batching='bucketing', beam_size=5, bpe_codes='/data/gnmt/wmt16_de_en/bpe.32000', cov_penalty_factor=0.1, cuda=True, cudnn=True, dataset_dir='/data/gnmt/wmt16_de_en', decay_factor=0.5, decay_interval=None, decay_steps=4, dllog_file='train_log.json', dropout=0.2, env=True, epochs=1, eval=True, grad_clip=5.0, hidden_size=1024, init_scale=8192, intra_epoch_eval=0, keep_checkpoints=0, lang={'src': 'en', 'tgt': 'de'}, len_norm_const=5.0, len_norm_factor=0.6, local_rank=0, log_all_ranks=True, lr=0.002, math='fp16', num_buckets=5, num_layers=4, optimizer='Adam', optimizer_extra='{}', prealloc_mode='always', print_freq=10, rank=0, remain_steps=0.666, resume=None, save_all=False, save_dir='gnmt', save_freq=5000, seed=2, shard_size=80, share_embedding=True, smoothing=0.1, src_lang='en', start_epoch=0, target_bleu=None, target_perf=None, test_batch_size=128, test_loader_workers=0, test_max_length=150, test_min_length=0, test_src='/data/gnmt/wmt16_de_en/newstest2014.tok.bpe.32000.en', test_tgt='/data/gnmt/wmt16_de_en/newstest2014.de', tgt_lang='de', train_batch_size=256, train_global_batch_size=None, train_iter_size=1, train_loader_workers=8, train_max_length=50, train_max_size=None, train_min_length=0, train_src='/data/gnmt/wmt16_de_en/train.tok.clean.bpe.32000.en', train_tgt='/data/gnmt/wmt16_de_en/train.tok.clean.bpe.32000.de', upscale_interval=128, val_batch_size=64, val_loader_workers=0, val_max_length=125, val_min_length=0, val_src='/data/gnmt/wmt16_de_en/newstest_dev.tok.clean.bpe.32000.en', val_tgt='/data/gnmt/wmt16_de_en/newstest_dev.tok.clean.bpe.32000.de', vocab='/data/gnmt/wmt16_de_en/vocab.bpe.32000', warmup=1, warmup_steps=200)
+0: Using master seed from command line: 2
+0: Worker 0 is using worker seed: 242886303
+0: Building vocabulary from /data/gnmt/wmt16_de_en/vocab.bpe.32000
+0: Size of vocabulary: 31800
+0: Processing data from /data/gnmt/wmt16_de_en/train.tok.clean.bpe.32000.en
+0: Processing data from /data/gnmt/wmt16_de_en/train.tok.clean.bpe.32000.de
+0: Filtering data, min len: 0, max len: 50
+0: Pairs before: 160078, after: 148120
+0: Processing data from /data/gnmt/wmt16_de_en/newstest_dev.tok.clean.bpe.32000.en
+0: Processing data from /data/gnmt/wmt16_de_en/newstest_dev.tok.clean.bpe.32000.de
+0: Filtering data, min len: 0, max len: 125
+0: Pairs before: 5100, after: 5100
+0: Processing data from /data/gnmt/wmt16_de_en/newstest2014.tok.bpe.32000.en
+0: Filtering data, min len: 0, max len: 150
+0: Pairs before: 3003, after: 3003
+0: GNMT(
+  (encoder): ResidualRecurrentEncoder(
+    (rnn_layers): ModuleList(
+      (0): LSTM(1024, 1024, bidirectional=True)
+      (1): LSTM(2048, 1024)
+      (2): LSTM(1024, 1024)
+      (3): LSTM(1024, 1024)
+    )
+    (dropout): Dropout(p=0.2, inplace=False)
+    (embedder): Embedding(31800, 1024, padding_idx=0)
+  )
+  (decoder): ResidualRecurrentDecoder(
+    (att_rnn): RecurrentAttention(
+      (rnn): LSTM(1024, 1024)
+      (attn): BahdanauAttention(
+        (linear_q): Linear(in_features=1024, out_features=1024, bias=False)
+        (linear_k): Linear(in_features=1024, out_features=1024, bias=False)
+      )
+      (dropout): Dropout(p=0.2, inplace=False)
+    )
+    (rnn_layers): ModuleList(
+      (0): LSTM(2048, 1024)
+      (1): LSTM(2048, 1024)
+      (2): LSTM(2048, 1024)
+    )
+    (embedder): Embedding(31800, 1024, padding_idx=0)
+    (classifier): Classifier(
+      (classifier): Linear(in_features=1024, out_features=31800, bias=True)
+    )
+    (dropout): Dropout(p=0.2, inplace=False)
+  )
+)
+0: Building LabelSmoothingLoss (smoothing: 0.1)
+0: Training optimizer config: {'optimizer': 'Adam', 'lr': 0.002}
+0: Training LR schedule config: {'warmup_steps': 200, 'remain_steps': 0.666, 'decay_interval': None, 'decay_steps': 4, 'decay_factor': 0.5}
+0: Number of parameters: 159605817
+0: Saving state of the tokenizer
+0: Using optimizer: Adam (
+Parameter Group 0
+    amsgrad: False
+    betas: (0.9, 0.999)
+    capturable: False
+    eps: 1e-08
+    foreach: None
+    lr: 0.002
+    maximize: False
+    weight_decay: 0
+)
+0: Scheduler warmup steps: 200
+0: Scheduler remain steps: 383
+0: Scheduler decay interval: 48
+0: Scheduler decay factor: 0.5
+0: Scheduler max decay steps: 4
+Selected optimization level O2:  FP16 training with FP32 batchnorm and FP32 master weights.
+
+Defaults for this optimization level are:
+enabled                : True
+opt_level              : O2
+cast_model_type        : torch.float16
+patch_torch_functions  : False
+keep_batchnorm_fp32    : True
+master_weights         : True
+loss_scale             : dynamic
+Processing user overrides (additional kwargs that are not None)...
+After processing overrides, optimization options are:
+enabled                : True
+opt_level              : O2
+cast_model_type        : torch.float16
+patch_torch_functions  : False
+keep_batchnorm_fp32    : False
+master_weights         : True
+loss_scale             : dynamic
+0: Initializing amp optimizer
+0: Starting epoch 0
+0: Executing preallocation
+0: Sampler for epoch 0 uses seed 364522461
+/opt/conda/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:131: UserWarning: Seems like `optimizer.step()` has been overridden after learning rate scheduler initialization. Please, make sure to call `optimizer.step()` before `lr_scheduler.step()`. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
+  warnings.warn("Seems like `optimizer.step()` has been overridden after learning rate scheduler "
+Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 4096.0
+0: TRAIN [0][0/576]	Time 0.437 (0.000)	Data 2.87e-01 (0.00e+00)	Tok/s 26007 (0)	Loss/tok 10.5997 (10.5997)	LR 2.047e-05
+0: TRAIN [0][10/576]	Time 0.229 (0.180)	Data 3.93e-05 (4.91e-05)	Tok/s 70708 (67945)	Loss/tok 9.6446 (10.0967)	LR 2.576e-05
+0: TRAIN [0][20/576]	Time 0.170 (0.170)	Data 6.03e-05 (4.86e-05)	Tok/s 68906 (66799)	Loss/tok 9.2469 (9.7782)	LR 3.244e-05
+0: TRAIN [0][30/576]	Time 0.116 (0.177)	Data 4.27e-05 (4.78e-05)	Tok/s 61188 (66864)	Loss/tok 8.7584 (9.5419)	LR 4.083e-05
+0: TRAIN [0][40/576]	Time 0.228 (0.180)	Data 4.24e-05 (4.74e-05)	Tok/s 70269 (66772)	Loss/tok 8.7058 (9.3435)	LR 5.141e-05
+0: TRAIN [0][50/576]	Time 0.122 (0.179)	Data 4.74e-05 (4.77e-05)	Tok/s 55709 (65609)	Loss/tok 8.2594 (9.1964)	LR 6.472e-05
+0: TRAIN [0][60/576]	Time 0.237 (0.183)	Data 4.22e-05 (4.72e-05)	Tok/s 67451 (65394)	Loss/tok 8.3052 (9.0542)	LR 8.148e-05
+0: TRAIN [0][70/576]	Time 0.235 (0.182)	Data 3.98e-05 (4.73e-05)	Tok/s 68875 (65020)	Loss/tok 8.1599 (8.9352)	LR 1.026e-04
+Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 2048.0
+0: TRAIN [0][80/576]	Time 0.117 (0.184)	Data 4.65e-05 (4.72e-05)	Tok/s 60014 (65397)	Loss/tok 7.7211 (8.8337)	LR 1.291e-04
+0: TRAIN [0][90/576]	Time 0.235 (0.183)	Data 4.39e-05 (4.74e-05)	Tok/s 68646 (65361)	Loss/tok 8.0118 (8.7393)	LR 1.626e-04
+0: TRAIN [0][100/576]	Time 0.230 (0.180)	Data 4.29e-05 (4.72e-05)	Tok/s 69931 (65140)	Loss/tok 7.8976 (8.6614)	LR 2.047e-04
+0: TRAIN [0][110/576]	Time 0.228 (0.182)	Data 4.24e-05 (4.71e-05)	Tok/s 70403 (65371)	Loss/tok 7.7942 (8.5759)	LR 2.576e-04
+0: TRAIN [0][120/576]	Time 0.116 (0.179)	Data 4.60e-05 (4.71e-05)	Tok/s 59664 (65224)	Loss/tok 7.3898 (8.5176)	LR 3.244e-04
+0: TRAIN [0][130/576]	Time 0.168 (0.179)	Data 4.60e-05 (4.71e-05)	Tok/s 69316 (65324)	Loss/tok 7.5908 (8.4546)	LR 4.083e-04
+Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 1024.0
+0: TRAIN [0][140/576]	Time 0.171 (0.180)	Data 4.39e-05 (4.70e-05)	Tok/s 67425 (65532)	Loss/tok 7.6446 (8.3991)	LR 5.141e-04
+Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 512.0
+0: TRAIN [0][150/576]	Time 0.116 (0.179)	Data 4.67e-05 (4.69e-05)	Tok/s 59868 (65597)	Loss/tok 7.3743 (8.3560)	LR 6.472e-04
+0: TRAIN [0][160/576]	Time 0.169 (0.181)	Data 4.65e-05 (4.70e-05)	Tok/s 67989 (65671)	Loss/tok 7.5862 (8.3169)	LR 8.148e-04
+0: TRAIN [0][170/576]	Time 0.116 (0.181)	Data 4.63e-05 (4.70e-05)	Tok/s 58984 (65621)	Loss/tok 7.2839 (8.2840)	LR 1.026e-03
+0: TRAIN [0][180/576]	Time 0.302 (0.182)	Data 4.63e-05 (4.70e-05)	Tok/s 69237 (65753)	Loss/tok 7.8214 (8.2448)	LR 1.291e-03
+0: TRAIN [0][190/576]	Time 0.176 (0.182)	Data 5.15e-05 (4.69e-05)	Tok/s 65748 (65650)	Loss/tok 7.6218 (8.2152)	LR 1.626e-03
+0: TRAIN [0][200/576]	Time 0.309 (0.182)	Data 4.63e-05 (4.68e-05)	Tok/s 67587 (65542)	Loss/tok 7.6971 (8.1829)	LR 2.000e-03
+0: TRAIN [0][210/576]	Time 0.171 (0.182)	Data 4.36e-05 (4.67e-05)	Tok/s 67460 (65489)	Loss/tok 7.2428 (8.1477)	LR 2.000e-03
+0: TRAIN [0][220/576]	Time 0.229 (0.182)	Data 4.58e-05 (4.69e-05)	Tok/s 70531 (65542)	Loss/tok 7.3791 (8.1083)	LR 2.000e-03
+0: TRAIN [0][230/576]	Time 0.173 (0.182)	Data 4.43e-05 (4.69e-05)	Tok/s 66820 (65596)	Loss/tok 7.0348 (8.0661)	LR 2.000e-03
+0: TRAIN [0][240/576]	Time 0.235 (0.182)	Data 4.55e-05 (4.69e-05)	Tok/s 68819 (65651)	Loss/tok 7.0114 (8.0217)	LR 2.000e-03
+0: TRAIN [0][250/576]	Time 0.230 (0.184)	Data 4.41e-05 (4.68e-05)	Tok/s 69785 (65783)	Loss/tok 6.9502 (7.9706)	LR 2.000e-03
+0: TRAIN [0][260/576]	Time 0.171 (0.183)	Data 4.48e-05 (4.68e-05)	Tok/s 66979 (65743)	Loss/tok 6.7533 (7.9325)	LR 2.000e-03
+0: TRAIN [0][270/576]	Time 0.123 (0.184)	Data 4.32e-05 (4.68e-05)	Tok/s 57270 (65741)	Loss/tok 6.3597 (7.8853)	LR 2.000e-03
+0: TRAIN [0][280/576]	Time 0.303 (0.184)	Data 4.48e-05 (4.68e-05)	Tok/s 68330 (65593)	Loss/tok 6.9246 (7.8485)	LR 2.000e-03
+0: TRAIN [0][290/576]	Time 0.121 (0.184)	Data 5.03e-05 (4.67e-05)	Tok/s 56607 (65455)	Loss/tok 6.3140 (7.8124)	LR 2.000e-03
+0: TRAIN [0][300/576]	Time 0.179 (0.185)	Data 4.55e-05 (4.68e-05)	Tok/s 64481 (65433)	Loss/tok 6.3931 (7.7674)	LR 2.000e-03
+0: TRAIN [0][310/576]	Time 0.180 (0.184)	Data 4.96e-05 (4.68e-05)	Tok/s 63936 (65247)	Loss/tok 6.3653 (7.7323)	LR 2.000e-03
+0: TRAIN [0][320/576]	Time 0.120 (0.183)	Data 4.46e-05 (4.68e-05)	Tok/s 57818 (65022)	Loss/tok 6.0976 (7.7008)	LR 2.000e-03
+0: TRAIN [0][330/576]	Time 0.180 (0.183)	Data 4.89e-05 (4.68e-05)	Tok/s 64231 (64942)	Loss/tok 6.3232 (7.6619)	LR 2.000e-03
+0: TRAIN [0][340/576]	Time 0.179 (0.183)	Data 4.32e-05 (4.69e-05)	Tok/s 64461 (64824)	Loss/tok 6.2295 (7.6260)	LR 2.000e-03
+0: TRAIN [0][350/576]	Time 0.177 (0.184)	Data 4.91e-05 (4.69e-05)	Tok/s 65814 (64818)	Loss/tok 6.1078 (7.5817)	LR 2.000e-03
+0: TRAIN [0][360/576]	Time 0.176 (0.186)	Data 5.08e-05 (4.69e-05)	Tok/s 65117 (64813)	Loss/tok 6.1133 (7.5380)	LR 2.000e-03
+0: TRAIN [0][370/576]	Time 0.307 (0.185)	Data 4.43e-05 (4.69e-05)	Tok/s 68193 (64674)	Loss/tok 6.3396 (7.5054)	LR 2.000e-03
+0: TRAIN [0][380/576]	Time 0.241 (0.185)	Data 4.53e-05 (4.68e-05)	Tok/s 67164 (64626)	Loss/tok 6.0917 (7.4656)	LR 2.000e-03
+0: TRAIN [0][390/576]	Time 0.178 (0.185)	Data 5.32e-05 (4.68e-05)	Tok/s 64804 (64625)	Loss/tok 5.8252 (7.4272)	LR 1.000e-03
+0: TRAIN [0][400/576]	Time 0.123 (0.186)	Data 4.36e-05 (4.69e-05)	Tok/s 57325 (64652)	Loss/tok 5.4757 (7.3842)	LR 1.000e-03
+0: TRAIN [0][410/576]	Time 0.123 (0.187)	Data 4.55e-05 (4.69e-05)	Tok/s 57560 (64650)	Loss/tok 5.4172 (7.3431)	LR 1.000e-03
+0: TRAIN [0][420/576]	Time 0.313 (0.187)	Data 4.79e-05 (4.69e-05)	Tok/s 66751 (64627)	Loss/tok 6.0168 (7.3036)	LR 1.000e-03
+0: TRAIN [0][430/576]	Time 0.177 (0.188)	Data 5.13e-05 (4.69e-05)	Tok/s 65073 (64600)	Loss/tok 5.5800 (7.2671)	LR 5.000e-04
+0: TRAIN [0][440/576]	Time 0.242 (0.187)	Data 4.46e-05 (4.68e-05)	Tok/s 66892 (64553)	Loss/tok 5.8014 (7.2334)	LR 5.000e-04
+0: TRAIN [0][450/576]	Time 0.239 (0.187)	Data 4.58e-05 (4.69e-05)	Tok/s 67596 (64571)	Loss/tok 5.7195 (7.1965)	LR 5.000e-04
+0: TRAIN [0][460/576]	Time 0.122 (0.187)	Data 4.63e-05 (4.70e-05)	Tok/s 56736 (64522)	Loss/tok 5.1162 (7.1633)	LR 5.000e-04
+0: TRAIN [0][470/576]	Time 0.242 (0.188)	Data 4.79e-05 (4.69e-05)	Tok/s 67137 (64495)	Loss/tok 5.6773 (7.1284)	LR 5.000e-04
+0: TRAIN [0][480/576]	Time 0.122 (0.188)	Data 4.82e-05 (4.70e-05)	Tok/s 55709 (64458)	Loss/tok 5.1072 (7.0957)	LR 2.500e-04
+0: TRAIN [0][490/576]	Time 0.178 (0.188)	Data 4.94e-05 (4.69e-05)	Tok/s 64376 (64376)	Loss/tok 5.3903 (7.0656)	LR 2.500e-04
+0: TRAIN [0][500/576]	Time 0.239 (0.188)	Data 4.65e-05 (4.69e-05)	Tok/s 67561 (64356)	Loss/tok 5.6329 (7.0331)	LR 2.500e-04
+0: TRAIN [0][510/576]	Time 0.123 (0.188)	Data 4.53e-05 (4.69e-05)	Tok/s 54829 (64284)	Loss/tok 5.0094 (7.0068)	LR 2.500e-04
+0: TRAIN [0][520/576]	Time 0.123 (0.187)	Data 4.89e-05 (4.68e-05)	Tok/s 57699 (64221)	Loss/tok 5.0379 (6.9814)	LR 2.500e-04
+0: TRAIN [0][530/576]	Time 0.179 (0.188)	Data 6.79e-05 (4.69e-05)	Tok/s 64588 (64205)	Loss/tok 5.3401 (6.9508)	LR 1.250e-04
+0: TRAIN [0][540/576]	Time 0.178 (0.188)	Data 4.51e-05 (4.69e-05)	Tok/s 64565 (64232)	Loss/tok 5.3507 (6.9191)	LR 1.250e-04
+Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 2048.0
+0: TRAIN [0][550/576]	Time 0.241 (0.188)	Data 4.51e-05 (4.69e-05)	Tok/s 67089 (64217)	Loss/tok 5.5323 (6.8918)	LR 1.250e-04
+0: TRAIN [0][560/576]	Time 0.176 (0.189)	Data 6.89e-04 (4.81e-05)	Tok/s 65612 (64250)	Loss/tok 5.2985 (6.8624)	LR 1.250e-04
+0: TRAIN [0][570/576]	Time 0.239 (0.189)	Data 2.60e-05 (4.78e-05)	Tok/s 67155 (64220)	Loss/tok 5.5196 (6.8376)	LR 1.250e-04
+0: Running validation on dev set
+0: Executing preallocation
+0: VALIDATION [0][0/80]	Time 0.075 (0.000)	Data 9.95e-04 (0.00e+00)	Tok/s 139040 (0)	Loss/tok 6.7337 (6.7337)
+0: VALIDATION [0][10/80]	Time 0.027 (0.034)	Data 7.90e-04 (8.19e-04)	Tok/s 219752 (205244)	Loss/tok 6.4669 (6.5541)
+0: VALIDATION [0][20/80]	Time 0.021 (0.029)	Data 7.09e-04 (7.91e-04)	Tok/s 220883 (209601)	Loss/tok 6.1983 (6.4741)
+0: VALIDATION [0][30/80]	Time 0.018 (0.026)	Data 7.02e-04 (7.72e-04)	Tok/s 217047 (211794)	Loss/tok 6.0610 (6.4064)
+0: VALIDATION [0][40/80]	Time 0.016 (0.023)	Data 7.13e-04 (7.58e-04)	Tok/s 207856 (211141)	Loss/tok 6.1352 (6.3675)
+0: VALIDATION [0][50/80]	Time 0.014 (0.022)	Data 7.25e-04 (7.52e-04)	Tok/s 186038 (210107)	Loss/tok 5.8788 (6.3341)
+0: VALIDATION [0][60/80]	Time 0.010 (0.020)	Data 7.30e-04 (7.45e-04)	Tok/s 205894 (209921)	Loss/tok 6.0993 (6.3072)
+0: VALIDATION [0][70/80]	Time 0.008 (0.018)	Data 7.04e-04 (7.40e-04)	Tok/s 207593 (209467)	Loss/tok 5.7344 (6.2762)
+0: Saving model to gnmt/model_best.pth
+0: Running evaluation on test set
+0: TEST [0][9/24]	Time 0.3200 (0.5669)	Decoder iters 149.0 (149.0)	Tok/s 26486 (24925)
+0: TEST [0][19/24]	Time 0.1917 (0.4060)	Decoder iters 97.0 (143.4)	Tok/s 22703 (25080)
+0: Running sacrebleu (parameters: --score-only -lc --tokenize intl)
+0: Finished evaluation on test set
+0: Summary: Epoch: 0	Training Loss: 6.8232	Validation Loss: 6.2475	Test BLEU: 1.41
+0: Performance: Epoch: 0	Training: 64241 Tok/s	Validation: 206413 Tok/s
+0: Finished epoch 0
+0: Total training time 129 s
+# Training Summary
+|**GPUs**|**Batch Size / GPU**|**Accuracy - FP16 (BLEU)**|**Throughput - FP16 (tok/s)**|**Time to Train - FP16 (min)**|
+|-------:|-------------------:|-------------------------:|----------------------------:|-----------------------------:|
+|       1|                 256|                      1.41|            64241.41689917352|             2.149888809521993|
+DONE!
diff --git a/pytorch/results/4090laptop_v1/PyTorch_gnmt_FP16/benchmark.para b/pytorch/results/4090laptop_v1/PyTorch_gnmt_FP16/benchmark.para
new file mode 100755
index 00000000..8df2db16
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_gnmt_FP16/benchmark.para
@@ -0,0 +1,2 @@
+GLOBAL_BATCH 256
+GPU 1
diff --git a/pytorch/results/4090laptop_v1/PyTorch_gnmt_FP32/12-07-2023_20-02-53.txt b/pytorch/results/4090laptop_v1/PyTorch_gnmt_FP32/12-07-2023_20-02-53.txt
new file mode 100755
index 00000000..7c93fa22
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_gnmt_FP32/12-07-2023_20-02-53.txt
@@ -0,0 +1,267 @@
+0: thread affinity: {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}
+0: Collecting environment information...
+0: PyTorch version: 1.13.0a0+d0d6b1f
+Is debug build: False
+CUDA used to build PyTorch: 11.8
+ROCM used to build PyTorch: N/A
+
+OS: Ubuntu 20.04.5 LTS (x86_64)
+GCC version: (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0
+Clang version: Could not collect
+CMake version: version 3.22.2
+Libc version: glibc-2.31
+
+Python version: 3.8.13 | packaged by conda-forge | (default, Mar 25 2022, 06:04:10)  [GCC 10.3.0] (64-bit runtime)
+Python platform: Linux-5.19.0-46-generic-x86_64-with-glibc2.10
+Is CUDA available: True
+CUDA runtime version: 11.8.89
+GPU models and configuration: GPU 0: NVIDIA GeForce RTX 4090 Laptop GPU
+Nvidia driver version: 525.116.04
+cuDNN version: Probably one of the following:
+/usr/lib/x86_64-linux-gnu/libcudnn.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.6.0
+HIP runtime version: N/A
+MIOpen runtime version: N/A
+Is XNNPACK available: True
+
+Versions of relevant libraries:
+[pip3] functorch==0.3.0a0
+[pip3] numpy==1.22.2
+[pip3] pytorch-quantization==2.1.2
+[pip3] pytorch-transformers==1.1.0
+[pip3] torch==1.13.0a0+d0d6b1f
+[pip3] torch-tensorrt==1.3.0a0
+[pip3] torchtext==0.11.0a0
+[pip3] torchvision==0.14.0a0
+[conda] functorch                 0.3.0a0                  pypi_0    pypi
+[conda] mkl                       2020.4             h726a3e6_304    conda-forge
+[conda] mkl-include               2020.4             h726a3e6_304    conda-forge
+[conda] numpy                     1.22.2           py38h6ae9a64_0    conda-forge
+[conda] pytorch-quantization      2.1.2                    pypi_0    pypi
+[conda] pytorch-transformers      1.1.0                    pypi_0    pypi
+[conda] torch                     1.13.0a0+d0d6b1f          pypi_0    pypi
+[conda] torch-tensorrt            1.3.0a0                  pypi_0    pypi
+[conda] torchtext                 0.11.0a0                 pypi_0    pypi
+[conda] torchvision               0.14.0a0                 pypi_0    pypi
+0: Saving results to: gnmt
+0: Run arguments: Namespace(affinity='socket_unique_interleaved', batching='bucketing', beam_size=5, bpe_codes='/data/gnmt/wmt16_de_en/bpe.32000', cov_penalty_factor=0.1, cuda=True, cudnn=True, dataset_dir='/data/gnmt/wmt16_de_en', decay_factor=0.5, decay_interval=None, decay_steps=4, dllog_file='train_log.json', dropout=0.2, env=True, epochs=1, eval=True, grad_clip=5.0, hidden_size=1024, init_scale=8192, intra_epoch_eval=0, keep_checkpoints=0, lang={'src': 'en', 'tgt': 'de'}, len_norm_const=5.0, len_norm_factor=0.6, local_rank=0, log_all_ranks=True, lr=0.002, math='fp32', num_buckets=5, num_layers=4, optimizer='Adam', optimizer_extra='{}', prealloc_mode='always', print_freq=10, rank=0, remain_steps=0.666, resume=None, save_all=False, save_dir='gnmt', save_freq=5000, seed=2, shard_size=80, share_embedding=True, smoothing=0.1, src_lang='en', start_epoch=0, target_bleu=None, target_perf=None, test_batch_size=128, test_loader_workers=0, test_max_length=150, test_min_length=0, test_src='/data/gnmt/wmt16_de_en/newstest2014.tok.bpe.32000.en', test_tgt='/data/gnmt/wmt16_de_en/newstest2014.de', tgt_lang='de', train_batch_size=128, train_global_batch_size=None, train_iter_size=1, train_loader_workers=8, train_max_length=50, train_max_size=None, train_min_length=0, train_src='/data/gnmt/wmt16_de_en/train.tok.clean.bpe.32000.en', train_tgt='/data/gnmt/wmt16_de_en/train.tok.clean.bpe.32000.de', upscale_interval=128, val_batch_size=64, val_loader_workers=0, val_max_length=125, val_min_length=0, val_src='/data/gnmt/wmt16_de_en/newstest_dev.tok.clean.bpe.32000.en', val_tgt='/data/gnmt/wmt16_de_en/newstest_dev.tok.clean.bpe.32000.de', vocab='/data/gnmt/wmt16_de_en/vocab.bpe.32000', warmup=1, warmup_steps=200)
+0: Using master seed from command line: 2
+0: Worker 0 is using worker seed: 242886303
+0: Building vocabulary from /data/gnmt/wmt16_de_en/vocab.bpe.32000
+0: Size of vocabulary: 31794
+0: Processing data from /data/gnmt/wmt16_de_en/train.tok.clean.bpe.32000.en
+0: Processing data from /data/gnmt/wmt16_de_en/train.tok.clean.bpe.32000.de
+0: Filtering data, min len: 0, max len: 50
+0: Pairs before: 160078, after: 148120
+0: Processing data from /data/gnmt/wmt16_de_en/newstest_dev.tok.clean.bpe.32000.en
+0: Processing data from /data/gnmt/wmt16_de_en/newstest_dev.tok.clean.bpe.32000.de
+0: Filtering data, min len: 0, max len: 125
+0: Pairs before: 5100, after: 5100
+0: Processing data from /data/gnmt/wmt16_de_en/newstest2014.tok.bpe.32000.en
+0: Filtering data, min len: 0, max len: 150
+0: Pairs before: 3003, after: 3003
+0: GNMT(
+  (encoder): ResidualRecurrentEncoder(
+    (rnn_layers): ModuleList(
+      (0): LSTM(1024, 1024, bidirectional=True)
+      (1): LSTM(2048, 1024)
+      (2): LSTM(1024, 1024)
+      (3): LSTM(1024, 1024)
+    )
+    (dropout): Dropout(p=0.2, inplace=False)
+    (embedder): Embedding(31794, 1024, padding_idx=0)
+  )
+  (decoder): ResidualRecurrentDecoder(
+    (att_rnn): RecurrentAttention(
+      (rnn): LSTM(1024, 1024)
+      (attn): BahdanauAttention(
+        (linear_q): Linear(in_features=1024, out_features=1024, bias=False)
+        (linear_k): Linear(in_features=1024, out_features=1024, bias=False)
+      )
+      (dropout): Dropout(p=0.2, inplace=False)
+    )
+    (rnn_layers): ModuleList(
+      (0): LSTM(2048, 1024)
+      (1): LSTM(2048, 1024)
+      (2): LSTM(2048, 1024)
+    )
+    (embedder): Embedding(31794, 1024, padding_idx=0)
+    (classifier): Classifier(
+      (classifier): Linear(in_features=1024, out_features=31794, bias=True)
+    )
+    (dropout): Dropout(p=0.2, inplace=False)
+  )
+)
+0: Building LabelSmoothingLoss (smoothing: 0.1)
+0: Training optimizer config: {'optimizer': 'Adam', 'lr': 0.002}
+0: Training LR schedule config: {'warmup_steps': 200, 'remain_steps': 0.666, 'decay_interval': None, 'decay_steps': 4, 'decay_factor': 0.5}
+0: Number of parameters: 159593523
+0: Saving state of the tokenizer
+0: Initializing fp32 optimizer
+0: Using optimizer: Adam (
+Parameter Group 0
+    amsgrad: False
+    betas: (0.9, 0.999)
+    capturable: False
+    eps: 1e-08
+    foreach: None
+    lr: 0.002
+    maximize: False
+    weight_decay: 0
+)
+0: Scheduler warmup steps: 200
+0: Scheduler remain steps: 768
+0: Scheduler decay interval: 96
+0: Scheduler decay factor: 0.5
+0: Scheduler max decay steps: 4
+0: Starting epoch 0
+0: Executing preallocation
+0: Sampler for epoch 0 uses seed 364522461
+/opt/conda/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:138: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
+  warnings.warn("Detected call of `lr_scheduler.step()` before `optimizer.step()`. "
+0: TRAIN [0][0/1154]	Time 0.524 (0.000)	Data 3.25e-01 (0.00e+00)	Tok/s 15359 (0)	Loss/tok 10.7126 (10.7126)	LR 2.047e-05
+0: TRAIN [0][10/1154]	Time 0.160 (0.189)	Data 4.29e-05 (3.70e-05)	Tok/s 35203 (41214)	Loss/tok 9.7172 (10.1311)	LR 2.576e-05
+0: TRAIN [0][20/1154]	Time 0.162 (0.171)	Data 3.41e-05 (3.46e-05)	Tok/s 35010 (40974)	Loss/tok 9.2519 (9.8263)	LR 3.244e-05
+0: TRAIN [0][30/1154]	Time 0.220 (0.166)	Data 3.24e-05 (3.45e-05)	Tok/s 25555 (39608)	Loss/tok 8.9936 (9.6001)	LR 4.083e-05
+0: TRAIN [0][40/1154]	Time 0.161 (0.169)	Data 3.77e-05 (3.46e-05)	Tok/s 36063 (39490)	Loss/tok 8.6678 (9.4126)	LR 5.141e-05
+0: TRAIN [0][50/1154]	Time 0.161 (0.175)	Data 3.93e-05 (3.49e-05)	Tok/s 35783 (39583)	Loss/tok 8.5326 (9.2548)	LR 6.472e-05
+0: TRAIN [0][60/1154]	Time 0.216 (0.173)	Data 3.41e-05 (3.51e-05)	Tok/s 37459 (39485)	Loss/tok 8.4422 (9.1266)	LR 8.148e-05
+0: TRAIN [0][70/1154]	Time 0.220 (0.176)	Data 3.60e-05 (3.53e-05)	Tok/s 26204 (39599)	Loss/tok 8.1085 (8.9903)	LR 1.026e-04
+0: TRAIN [0][80/1154]	Time 0.162 (0.178)	Data 3.65e-05 (3.54e-05)	Tok/s 35815 (39326)	Loss/tok 7.9546 (8.8692)	LR 1.291e-04
+0: TRAIN [0][90/1154]	Time 0.164 (0.177)	Data 3.19e-05 (3.55e-05)	Tok/s 35136 (39026)	Loss/tok 8.0720 (8.7841)	LR 1.626e-04
+0: TRAIN [0][100/1154]	Time 0.312 (0.179)	Data 3.65e-05 (3.57e-05)	Tok/s 33192 (38805)	Loss/tok 7.9843 (8.6949)	LR 2.047e-04
+0: TRAIN [0][110/1154]	Time 0.040 (0.177)	Data 3.50e-05 (3.58e-05)	Tok/s 43167 (38877)	Loss/tok 7.1626 (8.6280)	LR 2.576e-04
+0: TRAIN [0][120/1154]	Time 0.287 (0.180)	Data 3.48e-05 (3.59e-05)	Tok/s 36965 (38623)	Loss/tok 7.8968 (8.5494)	LR 3.244e-04
+0: TRAIN [0][130/1154]	Time 0.220 (0.179)	Data 3.39e-05 (3.59e-05)	Tok/s 26566 (38612)	Loss/tok 7.7115 (8.4967)	LR 4.083e-04
+0: TRAIN [0][140/1154]	Time 0.056 (0.179)	Data 5.22e-05 (3.60e-05)	Tok/s 64184 (38777)	Loss/tok 7.4547 (8.4432)	LR 5.141e-04
+0: TRAIN [0][150/1154]	Time 0.288 (0.182)	Data 4.63e-05 (3.62e-05)	Tok/s 36108 (38828)	Loss/tok 7.9362 (8.3952)	LR 6.472e-04
+0: TRAIN [0][160/1154]	Time 0.056 (0.182)	Data 3.58e-05 (3.63e-05)	Tok/s 62205 (38851)	Loss/tok 7.4675 (8.3531)	LR 8.148e-04
+0: TRAIN [0][170/1154]	Time 0.285 (0.182)	Data 3.72e-05 (3.65e-05)	Tok/s 36627 (38682)	Loss/tok 7.8243 (8.3175)	LR 1.026e-03
+0: TRAIN [0][180/1154]	Time 0.163 (0.182)	Data 3.81e-05 (3.66e-05)	Tok/s 34683 (38626)	Loss/tok 7.4898 (8.2805)	LR 1.291e-03
+0: TRAIN [0][190/1154]	Time 0.162 (0.182)	Data 3.84e-05 (3.65e-05)	Tok/s 34945 (38613)	Loss/tok 7.3912 (8.2472)	LR 1.626e-03
+0: TRAIN [0][200/1154]	Time 0.164 (0.184)	Data 3.48e-05 (3.65e-05)	Tok/s 35429 (38517)	Loss/tok 7.4980 (8.2101)	LR 2.000e-03
+0: TRAIN [0][210/1154]	Time 0.220 (0.184)	Data 3.65e-05 (3.65e-05)	Tok/s 26679 (38662)	Loss/tok 7.3607 (8.1757)	LR 2.000e-03
+0: TRAIN [0][220/1154]	Time 0.057 (0.183)	Data 4.12e-05 (3.65e-05)	Tok/s 63128 (38819)	Loss/tok 7.0779 (8.1400)	LR 2.000e-03
+0: TRAIN [0][230/1154]	Time 0.222 (0.181)	Data 3.27e-05 (3.65e-05)	Tok/s 25945 (38676)	Loss/tok 7.3041 (8.1140)	LR 2.000e-03
+0: TRAIN [0][240/1154]	Time 0.220 (0.180)	Data 5.65e-05 (3.65e-05)	Tok/s 26056 (38643)	Loss/tok 7.1596 (8.0815)	LR 2.000e-03
+0: TRAIN [0][250/1154]	Time 0.057 (0.179)	Data 3.39e-05 (3.65e-05)	Tok/s 60132 (38809)	Loss/tok 6.7184 (8.0509)	LR 2.000e-03
+0: TRAIN [0][260/1154]	Time 0.114 (0.179)	Data 4.08e-05 (3.66e-05)	Tok/s 31431 (38907)	Loss/tok 6.8098 (8.0181)	LR 2.000e-03
+0: TRAIN [0][270/1154]	Time 0.057 (0.181)	Data 3.65e-05 (3.67e-05)	Tok/s 61992 (38897)	Loss/tok 6.6663 (7.9747)	LR 2.000e-03
+0: TRAIN [0][280/1154]	Time 0.216 (0.182)	Data 4.12e-05 (3.68e-05)	Tok/s 37507 (38894)	Loss/tok 7.0500 (7.9386)	LR 2.000e-03
+0: TRAIN [0][290/1154]	Time 0.221 (0.183)	Data 3.31e-05 (3.68e-05)	Tok/s 36162 (38790)	Loss/tok 6.9138 (7.8987)	LR 2.000e-03
+0: TRAIN [0][300/1154]	Time 0.275 (0.183)	Data 3.55e-05 (3.68e-05)	Tok/s 28891 (38793)	Loss/tok 7.0007 (7.8656)	LR 2.000e-03
+0: TRAIN [0][310/1154]	Time 0.165 (0.183)	Data 3.67e-05 (3.69e-05)	Tok/s 35621 (38848)	Loss/tok 6.7055 (7.8322)	LR 2.000e-03
+0: TRAIN [0][320/1154]	Time 0.164 (0.182)	Data 3.39e-05 (3.68e-05)	Tok/s 35194 (38802)	Loss/tok 6.6214 (7.8037)	LR 2.000e-03
+0: TRAIN [0][330/1154]	Time 0.280 (0.181)	Data 3.72e-05 (3.69e-05)	Tok/s 28921 (38763)	Loss/tok 6.7852 (7.7704)	LR 2.000e-03
+0: TRAIN [0][340/1154]	Time 0.221 (0.181)	Data 3.91e-05 (3.69e-05)	Tok/s 36676 (38755)	Loss/tok 6.6614 (7.7383)	LR 2.000e-03
+0: TRAIN [0][350/1154]	Time 0.217 (0.181)	Data 3.60e-05 (3.69e-05)	Tok/s 36900 (38735)	Loss/tok 6.6283 (7.7055)	LR 2.000e-03
+0: TRAIN [0][360/1154]	Time 0.220 (0.180)	Data 3.43e-05 (3.69e-05)	Tok/s 25543 (38712)	Loss/tok 6.4443 (7.6757)	LR 2.000e-03
+0: TRAIN [0][370/1154]	Time 0.057 (0.180)	Data 3.67e-05 (3.69e-05)	Tok/s 61607 (38697)	Loss/tok 5.9913 (7.6449)	LR 2.000e-03
+0: TRAIN [0][380/1154]	Time 0.223 (0.180)	Data 3.48e-05 (3.69e-05)	Tok/s 25848 (38635)	Loss/tok 6.3240 (7.6114)	LR 2.000e-03
+0: TRAIN [0][390/1154]	Time 0.274 (0.180)	Data 3.46e-05 (3.70e-05)	Tok/s 29211 (38652)	Loss/tok 6.4285 (7.5822)	LR 2.000e-03
+0: TRAIN [0][400/1154]	Time 0.056 (0.180)	Data 4.22e-05 (3.69e-05)	Tok/s 59933 (38682)	Loss/tok 6.0218 (7.5531)	LR 2.000e-03
+0: TRAIN [0][410/1154]	Time 0.290 (0.181)	Data 4.17e-05 (3.70e-05)	Tok/s 36214 (38592)	Loss/tok 6.5010 (7.5183)	LR 2.000e-03
+0: TRAIN [0][420/1154]	Time 0.166 (0.181)	Data 3.84e-05 (3.71e-05)	Tok/s 35385 (38606)	Loss/tok 6.1184 (7.4878)	LR 2.000e-03
+0: TRAIN [0][430/1154]	Time 0.219 (0.181)	Data 3.93e-05 (3.71e-05)	Tok/s 36892 (38571)	Loss/tok 6.2865 (7.4567)	LR 2.000e-03
+0: TRAIN [0][440/1154]	Time 0.220 (0.181)	Data 4.17e-05 (3.72e-05)	Tok/s 37201 (38607)	Loss/tok 6.1260 (7.4267)	LR 2.000e-03
+0: TRAIN [0][450/1154]	Time 0.116 (0.180)	Data 3.50e-05 (3.72e-05)	Tok/s 29303 (38660)	Loss/tok 5.6314 (7.3989)	LR 2.000e-03
+0: TRAIN [0][460/1154]	Time 0.083 (0.180)	Data 3.65e-05 (3.72e-05)	Tok/s 39862 (38639)	Loss/tok 5.6894 (7.3724)	LR 2.000e-03
+0: TRAIN [0][470/1154]	Time 0.288 (0.181)	Data 3.39e-05 (3.73e-05)	Tok/s 36824 (38607)	Loss/tok 6.2084 (7.3383)	LR 2.000e-03
+0: TRAIN [0][480/1154]	Time 0.288 (0.181)	Data 4.43e-05 (3.74e-05)	Tok/s 36393 (38580)	Loss/tok 6.1537 (7.3077)	LR 2.000e-03
+0: TRAIN [0][490/1154]	Time 0.057 (0.181)	Data 3.98e-05 (3.74e-05)	Tok/s 60984 (38676)	Loss/tok 5.3169 (7.2831)	LR 2.000e-03
+0: TRAIN [0][500/1154]	Time 0.288 (0.181)	Data 4.03e-05 (3.75e-05)	Tok/s 36113 (38679)	Loss/tok 6.0950 (7.2556)	LR 2.000e-03
+0: TRAIN [0][510/1154]	Time 0.342 (0.181)	Data 3.19e-05 (3.75e-05)	Tok/s 30409 (38693)	Loss/tok 6.2852 (7.2297)	LR 2.000e-03
+0: TRAIN [0][520/1154]	Time 0.057 (0.181)	Data 4.03e-05 (3.75e-05)	Tok/s 61522 (38721)	Loss/tok 5.4829 (7.2051)	LR 2.000e-03
+0: TRAIN [0][530/1154]	Time 0.220 (0.181)	Data 3.79e-05 (3.75e-05)	Tok/s 36996 (38649)	Loss/tok 5.7444 (7.1764)	LR 2.000e-03
+0: TRAIN [0][540/1154]	Time 0.219 (0.181)	Data 3.41e-05 (3.75e-05)	Tok/s 36817 (38620)	Loss/tok 5.8598 (7.1497)	LR 2.000e-03
+0: TRAIN [0][550/1154]	Time 0.222 (0.181)	Data 3.58e-05 (3.76e-05)	Tok/s 26614 (38599)	Loss/tok 5.5235 (7.1219)	LR 2.000e-03
+0: TRAIN [0][560/1154]	Time 0.222 (0.181)	Data 3.27e-05 (3.76e-05)	Tok/s 26277 (38605)	Loss/tok 5.6844 (7.0985)	LR 2.000e-03
+0: TRAIN [0][570/1154]	Time 0.040 (0.181)	Data 3.70e-05 (3.75e-05)	Tok/s 43751 (38692)	Loss/tok 4.6782 (7.0758)	LR 2.000e-03
+0: TRAIN [0][580/1154]	Time 0.165 (0.180)	Data 3.79e-05 (3.75e-05)	Tok/s 35027 (38631)	Loss/tok 5.4971 (7.0523)	LR 2.000e-03
+0: TRAIN [0][590/1154]	Time 0.057 (0.180)	Data 4.48e-05 (3.76e-05)	Tok/s 61188 (38647)	Loss/tok 5.0812 (7.0269)	LR 2.000e-03
+0: TRAIN [0][600/1154]	Time 0.166 (0.180)	Data 3.60e-05 (3.75e-05)	Tok/s 34758 (38581)	Loss/tok 5.4423 (7.0051)	LR 2.000e-03
+0: TRAIN [0][610/1154]	Time 0.218 (0.180)	Data 3.53e-05 (3.76e-05)	Tok/s 36629 (38531)	Loss/tok 5.5784 (6.9796)	LR 2.000e-03
+0: TRAIN [0][620/1154]	Time 0.114 (0.180)	Data 3.58e-05 (3.76e-05)	Tok/s 30363 (38539)	Loss/tok 5.0091 (6.9599)	LR 2.000e-03
+0: TRAIN [0][630/1154]	Time 0.113 (0.180)	Data 3.70e-05 (3.76e-05)	Tok/s 31122 (38533)	Loss/tok 5.1192 (6.9366)	LR 2.000e-03
+0: TRAIN [0][640/1154]	Time 0.277 (0.180)	Data 3.19e-05 (3.76e-05)	Tok/s 28832 (38532)	Loss/tok 5.3310 (6.9124)	LR 2.000e-03
+0: TRAIN [0][650/1154]	Time 0.223 (0.179)	Data 3.29e-05 (3.77e-05)	Tok/s 25887 (38504)	Loss/tok 5.2477 (6.8888)	LR 2.000e-03
+0: TRAIN [0][660/1154]	Time 0.221 (0.179)	Data 3.89e-05 (3.77e-05)	Tok/s 25840 (38525)	Loss/tok 5.1399 (6.8668)	LR 2.000e-03
+0: TRAIN [0][670/1154]	Time 0.289 (0.179)	Data 3.74e-05 (3.76e-05)	Tok/s 36353 (38531)	Loss/tok 5.6217 (6.8451)	LR 2.000e-03
+0: TRAIN [0][680/1154]	Time 0.289 (0.179)	Data 3.74e-05 (3.77e-05)	Tok/s 35636 (38541)	Loss/tok 5.5607 (6.8220)	LR 2.000e-03
+0: TRAIN [0][690/1154]	Time 0.220 (0.179)	Data 3.50e-05 (3.77e-05)	Tok/s 37144 (38501)	Loss/tok 5.3468 (6.7987)	LR 2.000e-03
+0: TRAIN [0][700/1154]	Time 0.057 (0.179)	Data 3.84e-05 (3.77e-05)	Tok/s 60917 (38565)	Loss/tok 4.7991 (6.7784)	LR 2.000e-03
+0: TRAIN [0][710/1154]	Time 0.219 (0.179)	Data 3.65e-05 (3.77e-05)	Tok/s 37168 (38551)	Loss/tok 5.2546 (6.7551)	LR 2.000e-03
+0: TRAIN [0][720/1154]	Time 0.067 (0.179)	Data 3.74e-05 (3.77e-05)	Tok/s 25963 (38533)	Loss/tok 4.5712 (6.7364)	LR 2.000e-03
+0: TRAIN [0][730/1154]	Time 0.166 (0.179)	Data 3.62e-05 (3.77e-05)	Tok/s 34475 (38511)	Loss/tok 4.9460 (6.7146)	LR 2.000e-03
+0: TRAIN [0][740/1154]	Time 0.165 (0.179)	Data 4.65e-05 (3.77e-05)	Tok/s 34649 (38482)	Loss/tok 5.1100 (6.6944)	LR 2.000e-03
+0: TRAIN [0][750/1154]	Time 0.220 (0.179)	Data 3.50e-05 (3.77e-05)	Tok/s 36602 (38444)	Loss/tok 5.3448 (6.6759)	LR 2.000e-03
+0: TRAIN [0][760/1154]	Time 0.057 (0.178)	Data 3.55e-05 (3.77e-05)	Tok/s 62976 (38447)	Loss/tok 4.6472 (6.6552)	LR 2.000e-03
+0: TRAIN [0][770/1154]	Time 0.351 (0.179)	Data 3.24e-05 (3.77e-05)	Tok/s 29977 (38445)	Loss/tok 5.4552 (6.6340)	LR 1.000e-03
+0: TRAIN [0][780/1154]	Time 0.057 (0.179)	Data 3.96e-05 (3.77e-05)	Tok/s 62197 (38443)	Loss/tok 4.5733 (6.6137)	LR 1.000e-03
+0: TRAIN [0][790/1154]	Time 0.167 (0.178)	Data 3.53e-05 (3.77e-05)	Tok/s 34855 (38431)	Loss/tok 4.8699 (6.5948)	LR 1.000e-03
+0: TRAIN [0][800/1154]	Time 0.166 (0.178)	Data 1.33e-04 (3.78e-05)	Tok/s 35312 (38448)	Loss/tok 4.7082 (6.5748)	LR 1.000e-03
+0: TRAIN [0][810/1154]	Time 0.224 (0.178)	Data 3.48e-05 (3.78e-05)	Tok/s 25723 (38420)	Loss/tok 4.8555 (6.5566)	LR 1.000e-03
+0: TRAIN [0][820/1154]	Time 0.293 (0.178)	Data 3.50e-05 (3.78e-05)	Tok/s 35988 (38427)	Loss/tok 5.1531 (6.5351)	LR 1.000e-03
+0: TRAIN [0][830/1154]	Time 0.163 (0.178)	Data 3.53e-05 (3.77e-05)	Tok/s 35500 (38435)	Loss/tok 4.6825 (6.5143)	LR 1.000e-03
+0: TRAIN [0][840/1154]	Time 0.220 (0.179)	Data 3.48e-05 (3.77e-05)	Tok/s 36613 (38445)	Loss/tok 4.8917 (6.4940)	LR 1.000e-03
+0: TRAIN [0][850/1154]	Time 0.057 (0.178)	Data 4.17e-05 (3.77e-05)	Tok/s 59932 (38481)	Loss/tok 4.4217 (6.4759)	LR 1.000e-03
+0: TRAIN [0][860/1154]	Time 0.163 (0.178)	Data 4.10e-05 (3.77e-05)	Tok/s 36203 (38445)	Loss/tok 4.7127 (6.4573)	LR 1.000e-03
+0: TRAIN [0][870/1154]	Time 0.279 (0.178)	Data 3.50e-05 (3.77e-05)	Tok/s 28961 (38441)	Loss/tok 4.8841 (6.4410)	LR 5.000e-04
+0: TRAIN [0][880/1154]	Time 0.056 (0.178)	Data 3.62e-05 (3.77e-05)	Tok/s 62928 (38472)	Loss/tok 4.2783 (6.4227)	LR 5.000e-04
+0: TRAIN [0][890/1154]	Time 0.057 (0.178)	Data 3.39e-05 (3.77e-05)	Tok/s 60649 (38500)	Loss/tok 4.3177 (6.4050)	LR 5.000e-04
+0: TRAIN [0][900/1154]	Time 0.041 (0.177)	Data 3.53e-05 (3.76e-05)	Tok/s 42140 (38505)	Loss/tok 3.9203 (6.3876)	LR 5.000e-04
+0: TRAIN [0][910/1154]	Time 0.220 (0.178)	Data 3.67e-05 (3.76e-05)	Tok/s 36772 (38476)	Loss/tok 4.7512 (6.3677)	LR 5.000e-04
+0: TRAIN [0][920/1154]	Time 0.287 (0.178)	Data 3.60e-05 (3.76e-05)	Tok/s 36666 (38498)	Loss/tok 4.9735 (6.3510)	LR 5.000e-04
+0: TRAIN [0][930/1154]	Time 0.057 (0.178)	Data 3.86e-05 (3.76e-05)	Tok/s 60723 (38538)	Loss/tok 4.4071 (6.3322)	LR 5.000e-04
+0: TRAIN [0][940/1154]	Time 0.351 (0.178)	Data 3.39e-05 (3.76e-05)	Tok/s 29674 (38525)	Loss/tok 5.0235 (6.3159)	LR 5.000e-04
+0: TRAIN [0][950/1154]	Time 0.280 (0.178)	Data 3.98e-05 (3.76e-05)	Tok/s 28565 (38531)	Loss/tok 4.7500 (6.2975)	LR 5.000e-04
+0: TRAIN [0][960/1154]	Time 0.221 (0.178)	Data 3.72e-05 (3.76e-05)	Tok/s 36634 (38518)	Loss/tok 4.7705 (6.2781)	LR 2.500e-04
+0: TRAIN [0][970/1154]	Time 0.287 (0.178)	Data 3.41e-05 (3.76e-05)	Tok/s 36092 (38529)	Loss/tok 4.8457 (6.2639)	LR 2.500e-04
+0: TRAIN [0][980/1154]	Time 0.289 (0.178)	Data 3.79e-05 (3.75e-05)	Tok/s 36669 (38523)	Loss/tok 4.9383 (6.2437)	LR 2.500e-04
+0: TRAIN [0][990/1154]	Time 0.286 (0.179)	Data 3.41e-05 (3.75e-05)	Tok/s 36435 (38532)	Loss/tok 4.9420 (6.2250)	LR 2.500e-04
+0: TRAIN [0][1000/1154]	Time 0.057 (0.178)	Data 3.50e-05 (3.75e-05)	Tok/s 62283 (38576)	Loss/tok 4.1396 (6.2108)	LR 2.500e-04
+0: TRAIN [0][1010/1154]	Time 0.115 (0.178)	Data 3.24e-05 (3.75e-05)	Tok/s 31104 (38559)	Loss/tok 4.0953 (6.1971)	LR 2.500e-04
+0: TRAIN [0][1020/1154]	Time 0.344 (0.178)	Data 3.46e-05 (3.75e-05)	Tok/s 29958 (38547)	Loss/tok 4.9368 (6.1819)	LR 2.500e-04
+0: TRAIN [0][1030/1154]	Time 0.219 (0.178)	Data 3.70e-05 (3.75e-05)	Tok/s 36471 (38536)	Loss/tok 4.5618 (6.1666)	LR 2.500e-04
+0: TRAIN [0][1040/1154]	Time 0.115 (0.178)	Data 3.34e-05 (3.75e-05)	Tok/s 30905 (38550)	Loss/tok 4.3516 (6.1529)	LR 2.500e-04
+0: TRAIN [0][1050/1154]	Time 0.224 (0.178)	Data 3.50e-05 (3.75e-05)	Tok/s 35856 (38513)	Loss/tok 4.5859 (6.1359)	LR 2.500e-04
+0: TRAIN [0][1060/1154]	Time 0.281 (0.178)	Data 3.34e-05 (3.75e-05)	Tok/s 29045 (38531)	Loss/tok 4.6572 (6.1223)	LR 1.250e-04
+0: TRAIN [0][1070/1154]	Time 0.291 (0.178)	Data 3.60e-05 (3.75e-05)	Tok/s 36004 (38517)	Loss/tok 4.8888 (6.1066)	LR 1.250e-04
+0: TRAIN [0][1080/1154]	Time 0.225 (0.178)	Data 3.31e-05 (3.75e-05)	Tok/s 26019 (38548)	Loss/tok 4.5529 (6.0940)	LR 1.250e-04
+0: TRAIN [0][1090/1154]	Time 0.166 (0.178)	Data 3.89e-05 (3.75e-05)	Tok/s 35067 (38535)	Loss/tok 4.4793 (6.0792)	LR 1.250e-04
+0: TRAIN [0][1100/1154]	Time 0.041 (0.178)	Data 3.34e-05 (3.75e-05)	Tok/s 41315 (38550)	Loss/tok 3.8990 (6.0680)	LR 1.250e-04
+0: TRAIN [0][1110/1154]	Time 0.164 (0.178)	Data 3.36e-05 (3.74e-05)	Tok/s 35056 (38534)	Loss/tok 4.4919 (6.0538)	LR 1.250e-04
+0: TRAIN [0][1120/1154]	Time 0.166 (0.178)	Data 4.05e-05 (3.74e-05)	Tok/s 34465 (38507)	Loss/tok 4.4449 (6.0401)	LR 1.250e-04
+0: TRAIN [0][1130/1154]	Time 0.290 (0.178)	Data 4.15e-05 (3.74e-05)	Tok/s 36079 (38480)	Loss/tok 4.8751 (6.0278)	LR 1.250e-04
+0: TRAIN [0][1140/1154]	Time 0.278 (0.178)	Data 2.31e-05 (3.82e-05)	Tok/s 29264 (38503)	Loss/tok 4.7065 (6.0147)	LR 1.250e-04
+0: TRAIN [0][1150/1154]	Time 0.056 (0.178)	Data 2.36e-05 (3.81e-05)	Tok/s 61304 (38533)	Loss/tok 4.1795 (6.0016)	LR 1.250e-04
+0: Running validation on dev set
+0: Executing preallocation
+0: VALIDATION [0][0/80]	Time 0.131 (0.000)	Data 1.07e-03 (0.00e+00)	Tok/s 79775 (0)	Loss/tok 6.0776 (6.0776)
+0: VALIDATION [0][10/80]	Time 0.047 (0.060)	Data 8.04e-04 (8.44e-04)	Tok/s 123330 (115674)	Loss/tok 5.7609 (5.8544)
+0: VALIDATION [0][20/80]	Time 0.040 (0.052)	Data 8.13e-04 (8.24e-04)	Tok/s 116330 (115511)	Loss/tok 5.4256 (5.7720)
+0: VALIDATION [0][30/80]	Time 0.033 (0.047)	Data 7.63e-04 (8.07e-04)	Tok/s 117326 (116708)	Loss/tok 5.3058 (5.6967)
+0: VALIDATION [0][40/80]	Time 0.027 (0.042)	Data 7.77e-04 (7.95e-04)	Tok/s 118933 (117234)	Loss/tok 5.4759 (5.6538)
+0: VALIDATION [0][50/80]	Time 0.023 (0.039)	Data 7.63e-04 (7.85e-04)	Tok/s 116763 (117274)	Loss/tok 5.2053 (5.6185)
+0: VALIDATION [0][60/80]	Time 0.019 (0.036)	Data 7.38e-04 (7.78e-04)	Tok/s 111515 (116682)	Loss/tok 5.2934 (5.5874)
+0: VALIDATION [0][70/80]	Time 0.014 (0.033)	Data 7.08e-04 (7.70e-04)	Tok/s 114231 (116217)	Loss/tok 4.9133 (5.5553)
+0: Saving model to gnmt/model_best.pth
+0: Running evaluation on test set
+0: TEST [0][9/24]	Time 0.5316 (0.8840)	Decoder iters 149.0 (149.0)	Tok/s 16598 (14945)
+0: TEST [0][19/24]	Time 0.2429 (0.6437)	Decoder iters 84.0 (143.4)	Tok/s 17747 (15318)
+0: Running sacrebleu (parameters: --score-only -lc --tokenize intl)
+0: Finished evaluation on test set
+0: Summary: Epoch: 0	Training Loss: 5.9980	Validation Loss: 5.5263	Test BLEU: 3.90
+0: Performance: Epoch: 0	Training: 38538 Tok/s	Validation: 114762 Tok/s
+0: Finished epoch 0
+0: Total training time 231 s
+# Training Summary
+|**GPUs**|**Batch Size / GPU**|**Accuracy - FP32 (BLEU)**|**Throughput - FP32 (tok/s)**|**Time to Train - FP32 (min)**|
+|-------:|-------------------:|-------------------------:|----------------------------:|-----------------------------:|
+|       1|                 128|                       3.9|           38538.238509226874|            3.8544432878494264|
+DONE!
diff --git a/pytorch/results/4090laptop_v1/PyTorch_gnmt_FP32/benchmark.para b/pytorch/results/4090laptop_v1/PyTorch_gnmt_FP32/benchmark.para
new file mode 100755
index 00000000..497e227d
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_gnmt_FP32/benchmark.para
@@ -0,0 +1,2 @@
+GLOBAL_BATCH 128
+GPU 1
diff --git a/pytorch/results/4090laptop_v1/PyTorch_ncf_FP16/12-07-2023_19-57-50.txt b/pytorch/results/4090laptop_v1/PyTorch_ncf_FP16/12-07-2023_19-57-50.txt
new file mode 100755
index 00000000..4f5ce571
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_ncf_FP16/12-07-2023_19-57-50.txt
@@ -0,0 +1,39 @@
+DLL 2023-07-12 19:57:51.310441 - PARAMETER data : /data/ncf/cache/ml-20m  feature_spec_file : feature_spec.yaml  epochs : 2  batch_size : 2560000  valid_batch_size : 1048576  factors : 64  layers : [256, 256, 128, 64]  negative_samples : 4  learning_rate : 0.0045  topk : 10  seed : None  threshold : 1.0  beta1 : 0.25  beta2 : 0.5  eps : 1e-08  dropout : 0.5  checkpoint_dir :   load_checkpoint_path : None  mode : train  grads_accumulated : 1  amp : True  log_path : log.json  world_size : 1  distributed : False  local_rank : 0 
+Selected optimization level O2:  FP16 training with FP32 batchnorm and FP32 master weights.
+
+Defaults for this optimization level are:
+enabled                : True
+opt_level              : O2
+cast_model_type        : torch.float16
+patch_torch_functions  : False
+keep_batchnorm_fp32    : True
+master_weights         : True
+loss_scale             : dynamic
+Processing user overrides (additional kwargs that are not None)...
+After processing overrides, optimization options are:
+enabled                : True
+opt_level              : O2
+cast_model_type        : torch.float16
+patch_torch_functions  : False
+keep_batchnorm_fp32    : False
+master_weights         : True
+loss_scale             : dynamic
+NeuMF(
+  (mf_user_embed): Embedding(138493, 64)
+  (mf_item_embed): Embedding(26744, 64)
+  (mlp_user_embed): Embedding(138493, 128)
+  (mlp_item_embed): Embedding(26744, 128)
+  (mlp): ModuleList(
+    (0): Linear(in_features=256, out_features=256, bias=True)
+    (1): Linear(in_features=256, out_features=128, bias=True)
+    (2): Linear(in_features=128, out_features=64, bias=True)
+  )
+  (final): Linear(in_features=128, out_features=1, bias=True)
+)
+31832577 parameters
+DLL 2023-07-12 19:57:58.103002 - (0,) train_throughput : 1.796e+07 samples/s hr@10 : 0.84019 None train_epoch_time : 5.530 s validation_epoch_time : 0.272 s eval_throughput : 5.150e+07 samples/s validation_loss : 0.13027 None train_loss : 0.21938 None
+New best hr!
+DLL 2023-07-12 19:58:03.930792 - (1,) train_throughput : 1.782e+07 samples/s hr@10 : 0.88678 None train_epoch_time : 5.574 s validation_epoch_time : 0.254 s eval_throughput : 5.507e+07 samples/s validation_loss : 0.08989 None train_loss : 0.19097 None
+New best hr!
+DLL 2023-07-12 19:58:03.930932 - () best_train_throughput : 17958300.694254622 samples/s best_eval_throughput : 55072698.756852046 samples/s mean_train_throughput : 17887844.630462416 samples/s mean_eval_throughput : 53286054.935690746 samples/s best_accuracy : 0.886781281364401 None best_epoch : 1 None time_to_target : 12.50693154335022 s time_to_best_model : 12.506867408752441 s validation_loss : 0.08989 None train_loss : 0.19097 None
+DONE!
diff --git a/pytorch/results/4090laptop_v1/PyTorch_ncf_FP16/benchmark.para b/pytorch/results/4090laptop_v1/PyTorch_ncf_FP16/benchmark.para
new file mode 100755
index 00000000..8a4ad8d2
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_ncf_FP16/benchmark.para
@@ -0,0 +1,2 @@
+GLOBAL_BATCH 2560000
+GPU 1
diff --git a/pytorch/results/4090laptop_v1/PyTorch_ncf_FP32/12-07-2023_19-50-49.txt b/pytorch/results/4090laptop_v1/PyTorch_ncf_FP32/12-07-2023_19-50-49.txt
new file mode 100755
index 00000000..8135f86a
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_ncf_FP32/12-07-2023_19-50-49.txt
@@ -0,0 +1,20 @@
+DLL 2023-07-12 19:50:50.093601 - PARAMETER data : /data/ncf/cache/ml-20m  feature_spec_file : feature_spec.yaml  epochs : 2  batch_size : 1280000  valid_batch_size : 1048576  factors : 64  layers : [256, 256, 128, 64]  negative_samples : 4  learning_rate : 0.0045  topk : 10  seed : None  threshold : 1.0  beta1 : 0.25  beta2 : 0.5  eps : 1e-08  dropout : 0.5  checkpoint_dir :   load_checkpoint_path : None  mode : train  grads_accumulated : 1  amp : False  log_path : log.json  world_size : 1  distributed : False  local_rank : 0 
+NeuMF(
+  (mf_user_embed): Embedding(138493, 64)
+  (mf_item_embed): Embedding(26744, 64)
+  (mlp_user_embed): Embedding(138493, 128)
+  (mlp_item_embed): Embedding(26744, 128)
+  (mlp): ModuleList(
+    (0): Linear(in_features=256, out_features=256, bias=True)
+    (1): Linear(in_features=256, out_features=128, bias=True)
+    (2): Linear(in_features=128, out_features=64, bias=True)
+  )
+  (final): Linear(in_features=128, out_features=1, bias=True)
+)
+31832577 parameters
+DLL 2023-07-12 19:51:00.731062 - (0,) train_throughput : 1.070e+07 samples/s hr@10 : 0.86728 None train_epoch_time : 9.279 s validation_epoch_time : 0.430 s eval_throughput : 3.254e+07 samples/s validation_loss : 0.09269 None train_loss : 0.20130 None
+New best hr!
+DLL 2023-07-12 19:51:10.542128 - (1,) train_throughput : 1.059e+07 samples/s hr@10 : 0.90763 None train_epoch_time : 9.377 s validation_epoch_time : 0.434 s eval_throughput : 3.219e+07 samples/s validation_loss : 0.10803 None train_loss : 0.17806 None
+New best hr!
+DLL 2023-07-12 19:51:10.542301 - () best_train_throughput : 10702434.266083347 samples/s best_eval_throughput : 32535717.289666265 samples/s mean_train_throughput : 10646833.680661544 samples/s mean_eval_throughput : 32365291.00711158 samples/s best_accuracy : 0.9076343208682027 None best_epoch : 1 None time_to_target : 20.339418411254883 s time_to_best_model : 20.339338302612305 s validation_loss : 0.10803 None train_loss : 0.17806 None
+DONE!
diff --git a/pytorch/results/4090laptop_v1/PyTorch_ncf_FP32/benchmark.para b/pytorch/results/4090laptop_v1/PyTorch_ncf_FP32/benchmark.para
new file mode 100755
index 00000000..b22f6809
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_ncf_FP32/benchmark.para
@@ -0,0 +1,2 @@
+GLOBAL_BATCH 1280000
+GPU 1
diff --git a/pytorch/results/4090laptop_v1/PyTorch_resnet50_AMP/12-07-2023_19-52-22.txt b/pytorch/results/4090laptop_v1/PyTorch_resnet50_AMP/12-07-2023_19-52-22.txt
new file mode 100755
index 00000000..c6f5ada7
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_resnet50_AMP/12-07-2023_19-52-22.txt
@@ -0,0 +1,217 @@
+Namespace(conv_init='fan_in', fused_se=True, last_bn_0_init=False, num_classes=1000, pretrained=False, pretrained_from_file=None, trt=False)
+Training process 0 affinity: {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}
+DLL 2023-07-12 19:52:25.021144 - PARAMETER data : /data/imagenet  data_backend : synthetic  interpolation : bilinear  arch : resnet50  workers : 5  prefetch : 2  epochs : 2  run_epochs : -1  early_stopping_patience : -1  image_size : None  batch_size : 256  optimizer_batch_size : -1  lr : 0.1  lr_schedule : step  end_lr : 0  warmup : 0  label_smoothing : 0.0  mixup : 0.0  optimizer : sgd  momentum : 0.9  weight_decay : 0.0001  bn_weight_decay : False  rmsprop_alpha : 0.9  rmsprop_eps : 0.001  nesterov : False  print_freq : 1  resume : None  static_loss_scale : 256.0  dynamic_loss_scale : False  prof : 100  amp : True  seed : None  gather_checkpoints : 0  raport_file : benchmark.json  evaluate : False  training_only : True  save_checkpoints : True  jit : no  checkpoint_filename : checkpoint.pth.tar  workspace : ./  memory_format : nchw  use_ema : None  augmentation : None  gpu_affinity : none  topk : 5  distributed : False  local_rank : 0  gpu : 0  world_size : 1 
+DLL 2023-07-12 19:52:25.021240 - PARAMETER model.num_classes : 1000  model.last_bn_0_init : False  model.conv_init : fan_in  model.trt : False  model.fused_se : True  model.pretrained_from_file : None  model.pretrained : False 
+ ! Weight decay NOT applied to BN parameters 
+98
+63
+RUNNING EPOCHS FROM 0 TO 2
+DLL 2023-07-12 19:52:28.438075 - Epoch: 0 Iteration: 1  train.loss : 7.07880 None train.compute_ips : 74.94 images/s train.total_ips : 74.94 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:28.804308 - Epoch: 0 Iteration: 2  train.loss : 6.24451 None train.compute_ips : 699.39 images/s train.total_ips : 699.04 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:29.178873 - Epoch: 0 Iteration: 3  train.loss : 5.79288 None train.compute_ips : 683.66 images/s train.total_ips : 683.50 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:29.550837 - Epoch: 0 Iteration: 4  train.loss : 5.64733 None train.compute_ips : 688.42 images/s train.total_ips : 688.29 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:29.920948 - Epoch: 0 Iteration: 5  train.loss : 5.84741 None train.compute_ips : 691.87 images/s train.total_ips : 691.73 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:30.295378 - Epoch: 0 Iteration: 6  train.loss : 5.94366 None train.compute_ips : 683.91 images/s train.total_ips : 683.78 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:30.667596 - Epoch: 0 Iteration: 7  train.loss : 5.81480 None train.compute_ips : 687.98 images/s train.total_ips : 687.85 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:31.039389 - Epoch: 0 Iteration: 8  train.loss : 5.75348 None train.compute_ips : 688.75 images/s train.total_ips : 688.60 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:31.411680 - Epoch: 0 Iteration: 9  train.loss : 5.61537 None train.compute_ips : 687.81 images/s train.total_ips : 687.68 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:31.783462 - Epoch: 0 Iteration: 10  train.loss : 6.91435 None train.compute_ips : 688.76 images/s train.total_ips : 688.63 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:32.156732 - Epoch: 0 Iteration: 11  train.loss : 7.90392 None train.compute_ips : 686.02 images/s train.total_ips : 685.87 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:32.529958 - Epoch: 0 Iteration: 12  train.loss : 9.53590 None train.compute_ips : 686.13 images/s train.total_ips : 686.00 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:32.902243 - Epoch: 0 Iteration: 13  train.loss : 9.90483 None train.compute_ips : 687.88 images/s train.total_ips : 687.69 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:33.275331 - Epoch: 0 Iteration: 14  train.loss : 8.81466 None train.compute_ips : 686.37 images/s train.total_ips : 686.22 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:33.647698 - Epoch: 0 Iteration: 15  train.loss : 8.09288 None train.compute_ips : 687.71 images/s train.total_ips : 687.56 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:34.017413 - Epoch: 0 Iteration: 16  train.loss : 7.67441 None train.compute_ips : 692.64 images/s train.total_ips : 692.47 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:34.388149 - Epoch: 0 Iteration: 17  train.loss : 7.51008 None train.compute_ips : 690.72 images/s train.total_ips : 690.58 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:34.759680 - Epoch: 0 Iteration: 18  train.loss : 7.27017 None train.compute_ips : 689.24 images/s train.total_ips : 689.10 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:35.129045 - Epoch: 0 Iteration: 19  train.loss : 7.45210 None train.compute_ips : 693.31 images/s train.total_ips : 693.13 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:35.499286 - Epoch: 0 Iteration: 20  train.loss : 7.40491 None train.compute_ips : 691.63 images/s train.total_ips : 691.49 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:35.870813 - Epoch: 0 Iteration: 21  train.loss : 7.29130 None train.compute_ips : 689.26 images/s train.total_ips : 689.10 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:36.238358 - Epoch: 0 Iteration: 22  train.loss : 6.82422 None train.compute_ips : 696.71 images/s train.total_ips : 696.57 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:36.606482 - Epoch: 0 Iteration: 23  train.loss : 7.52657 None train.compute_ips : 695.62 images/s train.total_ips : 695.47 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:36.977809 - Epoch: 0 Iteration: 24  train.loss : 6.27613 None train.compute_ips : 689.64 images/s train.total_ips : 689.47 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:37.345794 - Epoch: 0 Iteration: 25  train.loss : 7.07678 None train.compute_ips : 695.92 images/s train.total_ips : 695.74 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:37.716871 - Epoch: 0 Iteration: 26  train.loss : 6.98684 None train.compute_ips : 690.09 images/s train.total_ips : 689.94 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:38.088710 - Epoch: 0 Iteration: 27  train.loss : 6.71765 None train.compute_ips : 688.66 images/s train.total_ips : 688.52 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:38.457777 - Epoch: 0 Iteration: 28  train.loss : 6.21946 None train.compute_ips : 693.83 images/s train.total_ips : 693.69 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:38.826548 - Epoch: 0 Iteration: 29  train.loss : 6.25310 None train.compute_ips : 694.39 images/s train.total_ips : 694.26 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:39.196151 - Epoch: 0 Iteration: 30  train.loss : 6.48370 None train.compute_ips : 692.84 images/s train.total_ips : 692.69 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:39.564198 - Epoch: 0 Iteration: 31  train.loss : 6.18797 None train.compute_ips : 695.77 images/s train.total_ips : 695.61 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:39.934348 - Epoch: 0 Iteration: 32  train.loss : 5.83284 None train.compute_ips : 691.80 images/s train.total_ips : 691.66 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:40.304673 - Epoch: 0 Iteration: 33  train.loss : 5.65212 None train.compute_ips : 691.48 images/s train.total_ips : 691.34 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:40.674318 - Epoch: 0 Iteration: 34  train.loss : 5.62877 None train.compute_ips : 692.77 images/s train.total_ips : 692.62 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:41.044364 - Epoch: 0 Iteration: 35  train.loss : 5.71558 None train.compute_ips : 692.01 images/s train.total_ips : 691.85 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:41.412072 - Epoch: 0 Iteration: 36  train.loss : 5.53938 None train.compute_ips : 696.39 images/s train.total_ips : 696.25 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:41.780178 - Epoch: 0 Iteration: 37  train.loss : 5.49489 None train.compute_ips : 695.65 images/s train.total_ips : 695.52 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:42.149685 - Epoch: 0 Iteration: 38  train.loss : 5.46064 None train.compute_ips : 693.02 images/s train.total_ips : 692.86 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:42.518685 - Epoch: 0 Iteration: 39  train.loss : 5.43512 None train.compute_ips : 693.96 images/s train.total_ips : 693.82 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:42.888401 - Epoch: 0 Iteration: 40  train.loss : 5.40546 None train.compute_ips : 692.63 images/s train.total_ips : 692.48 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:43.258550 - Epoch: 0 Iteration: 41  train.loss : 5.37172 None train.compute_ips : 691.81 images/s train.total_ips : 691.66 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:43.625868 - Epoch: 0 Iteration: 42  train.loss : 5.33253 None train.compute_ips : 697.14 images/s train.total_ips : 696.99 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:43.994168 - Epoch: 0 Iteration: 43  train.loss : 5.28918 None train.compute_ips : 695.27 images/s train.total_ips : 695.13 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:44.362679 - Epoch: 0 Iteration: 44  train.loss : 5.24731 None train.compute_ips : 694.88 images/s train.total_ips : 694.74 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:44.732268 - Epoch: 0 Iteration: 45  train.loss : 5.20805 None train.compute_ips : 692.88 images/s train.total_ips : 692.73 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:45.101754 - Epoch: 0 Iteration: 46  train.loss : 5.16995 None train.compute_ips : 693.08 images/s train.total_ips : 692.90 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:45.473296 - Epoch: 0 Iteration: 47  train.loss : 5.13211 None train.compute_ips : 689.23 images/s train.total_ips : 689.08 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:45.842773 - Epoch: 0 Iteration: 48  train.loss : 5.09322 None train.compute_ips : 693.10 images/s train.total_ips : 692.93 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:46.210775 - Epoch: 0 Iteration: 49  train.loss : 5.05142 None train.compute_ips : 695.87 images/s train.total_ips : 695.71 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:46.578621 - Epoch: 0 Iteration: 50  train.loss : 5.00613 None train.compute_ips : 696.14 images/s train.total_ips : 696.00 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:46.947589 - Epoch: 0 Iteration: 51  train.loss : 4.95684 None train.compute_ips : 694.03 images/s train.total_ips : 693.88 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:47.317158 - Epoch: 0 Iteration: 52  train.loss : 4.90534 None train.compute_ips : 692.89 images/s train.total_ips : 692.75 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:47.688862 - Epoch: 0 Iteration: 53  train.loss : 4.85296 None train.compute_ips : 688.91 images/s train.total_ips : 688.78 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:48.059237 - Epoch: 0 Iteration: 54  train.loss : 4.79960 None train.compute_ips : 691.39 images/s train.total_ips : 691.24 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:48.427335 - Epoch: 0 Iteration: 55  train.loss : 4.74157 None train.compute_ips : 695.66 images/s train.total_ips : 695.52 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:48.796991 - Epoch: 0 Iteration: 56  train.loss : 4.68104 None train.compute_ips : 692.75 images/s train.total_ips : 692.59 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:49.168401 - Epoch: 0 Iteration: 57  train.loss : 4.61877 None train.compute_ips : 689.51 images/s train.total_ips : 689.35 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:49.536446 - Epoch: 0 Iteration: 58  train.loss : 4.55121 None train.compute_ips : 695.84 images/s train.total_ips : 695.63 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:49.906803 - Epoch: 0 Iteration: 59  train.loss : 4.48118 None train.compute_ips : 691.45 images/s train.total_ips : 691.27 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:50.278237 - Epoch: 0 Iteration: 60  train.loss : 4.40666 None train.compute_ips : 689.43 images/s train.total_ips : 689.29 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:50.647289 - Epoch: 0 Iteration: 61  train.loss : 4.33462 None train.compute_ips : 693.88 images/s train.total_ips : 693.72 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:51.015896 - Epoch: 0 Iteration: 62  train.loss : 4.25632 None train.compute_ips : 694.76 images/s train.total_ips : 694.60 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:51.385398 - Epoch: 0 Iteration: 63  train.loss : 4.17373 None train.compute_ips : 693.07 images/s train.total_ips : 692.87 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:51.753496 - Epoch: 0 Iteration: 64  train.loss : 4.09086 None train.compute_ips : 695.69 images/s train.total_ips : 695.52 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:52.125623 - Epoch: 0 Iteration: 65  train.loss : 4.00362 None train.compute_ips : 688.21 images/s train.total_ips : 688.06 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:52.497071 - Epoch: 0 Iteration: 66  train.loss : 3.91214 None train.compute_ips : 689.53 images/s train.total_ips : 689.26 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:52.865607 - Epoch: 0 Iteration: 67  train.loss : 3.82087 None train.compute_ips : 695.02 images/s train.total_ips : 694.75 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:53.236522 - Epoch: 0 Iteration: 68  train.loss : 3.72653 None train.compute_ips : 690.54 images/s train.total_ips : 690.28 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:53.605974 - Epoch: 0 Iteration: 69  train.loss : 3.62963 None train.compute_ips : 693.22 images/s train.total_ips : 693.01 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:53.974219 - Epoch: 0 Iteration: 70  train.loss : 3.53009 None train.compute_ips : 695.46 images/s train.total_ips : 695.25 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:54.345515 - Epoch: 0 Iteration: 71  train.loss : 3.42851 None train.compute_ips : 689.73 images/s train.total_ips : 689.54 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:54.716486 - Epoch: 0 Iteration: 72  train.loss : 3.32577 None train.compute_ips : 690.32 images/s train.total_ips : 690.14 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:55.087065 - Epoch: 0 Iteration: 73  train.loss : 3.22110 None train.compute_ips : 691.09 images/s train.total_ips : 690.92 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:55.459380 - Epoch: 0 Iteration: 74  train.loss : 3.11737 None train.compute_ips : 687.80 images/s train.total_ips : 687.61 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:55.828529 - Epoch: 0 Iteration: 75  train.loss : 3.01497 None train.compute_ips : 693.69 images/s train.total_ips : 693.54 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:56.195895 - Epoch: 0 Iteration: 76  train.loss : 2.91171 None train.compute_ips : 697.06 images/s train.total_ips : 696.91 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:56.567291 - Epoch: 0 Iteration: 77  train.loss : 2.80769 None train.compute_ips : 689.50 images/s train.total_ips : 689.34 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:56.937508 - Epoch: 0 Iteration: 78  train.loss : 2.70224 None train.compute_ips : 691.69 images/s train.total_ips : 691.54 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:57.309147 - Epoch: 0 Iteration: 79  train.loss : 2.59859 None train.compute_ips : 689.04 images/s train.total_ips : 688.89 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:57.680898 - Epoch: 0 Iteration: 80  train.loss : 2.49583 None train.compute_ips : 688.85 images/s train.total_ips : 688.69 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:58.051970 - Epoch: 0 Iteration: 81  train.loss : 2.39361 None train.compute_ips : 690.13 images/s train.total_ips : 689.97 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:58.420916 - Epoch: 0 Iteration: 82  train.loss : 2.29211 None train.compute_ips : 694.09 images/s train.total_ips : 693.91 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:58.791334 - Epoch: 0 Iteration: 83  train.loss : 2.19140 None train.compute_ips : 691.33 images/s train.total_ips : 691.16 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:59.160453 - Epoch: 0 Iteration: 84  train.loss : 2.09297 None train.compute_ips : 693.75 images/s train.total_ips : 693.60 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:59.529737 - Epoch: 0 Iteration: 85  train.loss : 1.99788 None train.compute_ips : 693.43 images/s train.total_ips : 693.28 images/s train.lr : 0.1 
+DLL 2023-07-12 19:52:59.902490 - Epoch: 0 Iteration: 86  train.loss : 1.90434 None train.compute_ips : 687.00 images/s train.total_ips : 686.85 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:00.274916 - Epoch: 0 Iteration: 87  train.loss : 1.81245 None train.compute_ips : 687.61 images/s train.total_ips : 687.44 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:00.643280 - Epoch: 0 Iteration: 88  train.loss : 1.72206 None train.compute_ips : 695.20 images/s train.total_ips : 695.02 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:01.013082 - Epoch: 0 Iteration: 89  train.loss : 1.63433 None train.compute_ips : 692.47 images/s train.total_ips : 692.32 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:01.382840 - Epoch: 0 Iteration: 90  train.loss : 1.55001 None train.compute_ips : 692.55 images/s train.total_ips : 692.40 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:01.751865 - Epoch: 0 Iteration: 91  train.loss : 1.47023 None train.compute_ips : 693.93 images/s train.total_ips : 693.77 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:02.123293 - Epoch: 0 Iteration: 92  train.loss : 1.39355 None train.compute_ips : 689.47 images/s train.total_ips : 689.31 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:02.495124 - Epoch: 0 Iteration: 93  train.loss : 1.32087 None train.compute_ips : 688.68 images/s train.total_ips : 688.51 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:02.865537 - Epoch: 0 Iteration: 94  train.loss : 1.25176 None train.compute_ips : 691.32 images/s train.total_ips : 691.18 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:03.235547 - Epoch: 0 Iteration: 95  train.loss : 1.18631 None train.compute_ips : 692.07 images/s train.total_ips : 691.92 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:03.606333 - Epoch: 0 Iteration: 96  train.loss : 1.12452 None train.compute_ips : 690.66 images/s train.total_ips : 690.48 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:03.975642 - Epoch: 0 Iteration: 97  train.loss : 1.06596 None train.compute_ips : 693.40 images/s train.total_ips : 693.24 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:04.346305 - Epoch: 0 Iteration: 98  train.loss : 1.01057 None train.compute_ips : 690.86 images/s train.total_ips : 690.70 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:04.717976 - Epoch: 0 Iteration: 99  train.loss : 0.95862 None train.compute_ips : 688.98 images/s train.total_ips : 688.84 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:10.095312 - Epoch: 0  train.loss : 4.65508 None train.compute_ips : 685.55 images/s train.total_ips : 685.39 images/s train.lr : 0.1  val.top1 : None % val.top5 : None % val.loss : None None val.compute_ips : None images/s val.total_ips : None images/s val.compute_latency : None s
+DLL 2023-07-12 19:53:10.095724 - Epoch: 0  train.data_time : 0.00009 s train.compute_time : 0.40056 s val.data_time : None s val.compute_latency_at100 : None s val.compute_latency_at99 : None s val.compute_latency_at95 : None s
+SAVING ./checkpoint_0000.pth.tar
+DLL 2023-07-12 19:53:10.655214 - Epoch: 1 Iteration: 1  train.loss : 0.86532 None train.compute_ips : 703.55 images/s train.total_ips : 703.53 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:11.018416 - Epoch: 1 Iteration: 2  train.loss : 0.82342 None train.compute_ips : 705.16 images/s train.total_ips : 704.92 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:11.385376 - Epoch: 1 Iteration: 3  train.loss : 0.78523 None train.compute_ips : 697.85 images/s train.total_ips : 697.67 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:11.756450 - Epoch: 1 Iteration: 4  train.loss : 0.74969 None train.compute_ips : 690.08 images/s train.total_ips : 689.94 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:12.122635 - Epoch: 1 Iteration: 5  train.loss : 0.71718 None train.compute_ips : 699.30 images/s train.total_ips : 699.15 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:12.488669 - Epoch: 1 Iteration: 6  train.loss : 0.68728 None train.compute_ips : 699.59 images/s train.total_ips : 699.44 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:12.861883 - Epoch: 1 Iteration: 7  train.loss : 0.65983 None train.compute_ips : 686.13 images/s train.total_ips : 685.98 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:13.229583 - Epoch: 1 Iteration: 8  train.loss : 0.63445 None train.compute_ips : 696.42 images/s train.total_ips : 696.27 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:13.594610 - Epoch: 1 Iteration: 9  train.loss : 0.61192 None train.compute_ips : 701.52 images/s train.total_ips : 701.37 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:13.965377 - Epoch: 1 Iteration: 10  train.loss : 0.59133 None train.compute_ips : 690.66 images/s train.total_ips : 690.51 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:14.335064 - Epoch: 1 Iteration: 11  train.loss : 0.57287 None train.compute_ips : 692.72 images/s train.total_ips : 692.53 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:14.703943 - Epoch: 1 Iteration: 12  train.loss : 0.55630 None train.compute_ips : 694.20 images/s train.total_ips : 694.05 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:15.074639 - Epoch: 1 Iteration: 13  train.loss : 0.54129 None train.compute_ips : 690.79 images/s train.total_ips : 690.65 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:15.445014 - Epoch: 1 Iteration: 14  train.loss : 0.52770 None train.compute_ips : 691.41 images/s train.total_ips : 691.25 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:15.810361 - Epoch: 1 Iteration: 15  train.loss : 0.51568 None train.compute_ips : 700.92 images/s train.total_ips : 700.76 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:16.180033 - Epoch: 1 Iteration: 16  train.loss : 0.50489 None train.compute_ips : 692.72 images/s train.total_ips : 692.56 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:16.548679 - Epoch: 1 Iteration: 17  train.loss : 0.49499 None train.compute_ips : 694.64 images/s train.total_ips : 694.48 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:16.916737 - Epoch: 1 Iteration: 18  train.loss : 0.48604 None train.compute_ips : 695.76 images/s train.total_ips : 695.60 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:17.287977 - Epoch: 1 Iteration: 19  train.loss : 0.47781 None train.compute_ips : 689.80 images/s train.total_ips : 689.64 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:17.657622 - Epoch: 1 Iteration: 20  train.loss : 0.47034 None train.compute_ips : 692.78 images/s train.total_ips : 692.60 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:18.026186 - Epoch: 1 Iteration: 21  train.loss : 0.46328 None train.compute_ips : 694.81 images/s train.total_ips : 694.66 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:18.393017 - Epoch: 1 Iteration: 22  train.loss : 0.45677 None train.compute_ips : 698.08 images/s train.total_ips : 697.93 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:18.760366 - Epoch: 1 Iteration: 23  train.loss : 0.45053 None train.compute_ips : 697.10 images/s train.total_ips : 696.93 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:19.130429 - Epoch: 1 Iteration: 24  train.loss : 0.44464 None train.compute_ips : 691.97 images/s train.total_ips : 691.82 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:19.501847 - Epoch: 1 Iteration: 25  train.loss : 0.43905 None train.compute_ips : 689.45 images/s train.total_ips : 689.30 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:19.870679 - Epoch: 1 Iteration: 26  train.loss : 0.43359 None train.compute_ips : 694.28 images/s train.total_ips : 694.13 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:20.242068 - Epoch: 1 Iteration: 27  train.loss : 0.42835 None train.compute_ips : 689.51 images/s train.total_ips : 689.36 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:20.609826 - Epoch: 1 Iteration: 28  train.loss : 0.42325 None train.compute_ips : 696.35 images/s train.total_ips : 696.20 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:20.977187 - Epoch: 1 Iteration: 29  train.loss : 0.41816 None train.compute_ips : 697.13 images/s train.total_ips : 696.92 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:21.346808 - Epoch: 1 Iteration: 30  train.loss : 0.41315 None train.compute_ips : 692.82 images/s train.total_ips : 692.65 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:21.717520 - Epoch: 1 Iteration: 31  train.loss : 0.40823 None train.compute_ips : 690.77 images/s train.total_ips : 690.61 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:22.087418 - Epoch: 1 Iteration: 32  train.loss : 0.40366 None train.compute_ips : 692.29 images/s train.total_ips : 692.13 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:22.457689 - Epoch: 1 Iteration: 33  train.loss : 0.39940 None train.compute_ips : 691.59 images/s train.total_ips : 691.44 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:22.826800 - Epoch: 1 Iteration: 34  train.loss : 0.39550 None train.compute_ips : 693.76 images/s train.total_ips : 693.61 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:23.193007 - Epoch: 1 Iteration: 35  train.loss : 0.39214 None train.compute_ips : 699.29 images/s train.total_ips : 699.11 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:23.561859 - Epoch: 1 Iteration: 36  train.loss : 0.38929 None train.compute_ips : 694.24 images/s train.total_ips : 694.10 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:23.930893 - Epoch: 1 Iteration: 37  train.loss : 0.38695 None train.compute_ips : 693.91 images/s train.total_ips : 693.75 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:24.300509 - Epoch: 1 Iteration: 38  train.loss : 0.38506 None train.compute_ips : 692.83 images/s train.total_ips : 692.66 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:24.669040 - Epoch: 1 Iteration: 39  train.loss : 0.38338 None train.compute_ips : 694.86 images/s train.total_ips : 694.70 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:25.040962 - Epoch: 1 Iteration: 40  train.loss : 0.38184 None train.compute_ips : 688.51 images/s train.total_ips : 688.37 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:25.406958 - Epoch: 1 Iteration: 41  train.loss : 0.38021 None train.compute_ips : 699.66 images/s train.total_ips : 699.51 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:25.774106 - Epoch: 1 Iteration: 42  train.loss : 0.37855 None train.compute_ips : 697.48 images/s train.total_ips : 697.33 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:26.143093 - Epoch: 1 Iteration: 43  train.loss : 0.37695 None train.compute_ips : 694.02 images/s train.total_ips : 693.84 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:26.512684 - Epoch: 1 Iteration: 44  train.loss : 0.37537 None train.compute_ips : 692.86 images/s train.total_ips : 692.71 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:26.883575 - Epoch: 1 Iteration: 45  train.loss : 0.37378 None train.compute_ips : 690.43 images/s train.total_ips : 690.28 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:27.253333 - Epoch: 1 Iteration: 46  train.loss : 0.37235 None train.compute_ips : 692.54 images/s train.total_ips : 692.39 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:27.621227 - Epoch: 1 Iteration: 47  train.loss : 0.37076 None train.compute_ips : 696.06 images/s train.total_ips : 695.90 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:27.987920 - Epoch: 1 Iteration: 48  train.loss : 0.36925 None train.compute_ips : 698.34 images/s train.total_ips : 698.18 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:28.357678 - Epoch: 1 Iteration: 49  train.loss : 0.36775 None train.compute_ips : 692.54 images/s train.total_ips : 692.40 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:28.724595 - Epoch: 1 Iteration: 50  train.loss : 0.36619 None train.compute_ips : 697.91 images/s train.total_ips : 697.76 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:29.095874 - Epoch: 1 Iteration: 51  train.loss : 0.36459 None train.compute_ips : 689.73 images/s train.total_ips : 689.56 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:29.466696 - Epoch: 1 Iteration: 52  train.loss : 0.36301 None train.compute_ips : 690.60 images/s train.total_ips : 690.40 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:29.835289 - Epoch: 1 Iteration: 53  train.loss : 0.36138 None train.compute_ips : 694.73 images/s train.total_ips : 694.59 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:30.203044 - Epoch: 1 Iteration: 54  train.loss : 0.35970 None train.compute_ips : 696.32 images/s train.total_ips : 696.17 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:30.570610 - Epoch: 1 Iteration: 55  train.loss : 0.35797 None train.compute_ips : 696.68 images/s train.total_ips : 696.53 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:30.937839 - Epoch: 1 Iteration: 56  train.loss : 0.35624 None train.compute_ips : 697.32 images/s train.total_ips : 697.17 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:31.307844 - Epoch: 1 Iteration: 57  train.loss : 0.35449 None train.compute_ips : 692.09 images/s train.total_ips : 691.94 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:31.679624 - Epoch: 1 Iteration: 58  train.loss : 0.35266 None train.compute_ips : 688.79 images/s train.total_ips : 688.63 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:32.047636 - Epoch: 1 Iteration: 59  train.loss : 0.35080 None train.compute_ips : 695.84 images/s train.total_ips : 695.68 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:32.416483 - Epoch: 1 Iteration: 60  train.loss : 0.34883 None train.compute_ips : 694.27 images/s train.total_ips : 694.11 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:32.785031 - Epoch: 1 Iteration: 61  train.loss : 0.34683 None train.compute_ips : 694.84 images/s train.total_ips : 694.68 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:33.154173 - Epoch: 1 Iteration: 62  train.loss : 0.34487 None train.compute_ips : 693.72 images/s train.total_ips : 693.55 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:33.522663 - Epoch: 1 Iteration: 63  train.loss : 0.34274 None train.compute_ips : 694.95 images/s train.total_ips : 694.78 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:33.895641 - Epoch: 1 Iteration: 64  train.loss : 0.34057 None train.compute_ips : 686.57 images/s train.total_ips : 686.43 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:34.265070 - Epoch: 1 Iteration: 65  train.loss : 0.33841 None train.compute_ips : 693.17 images/s train.total_ips : 693.01 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:34.633336 - Epoch: 1 Iteration: 66  train.loss : 0.33608 None train.compute_ips : 695.40 images/s train.total_ips : 695.24 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:35.002166 - Epoch: 1 Iteration: 67  train.loss : 0.33366 None train.compute_ips : 694.31 images/s train.total_ips : 694.14 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:35.370161 - Epoch: 1 Iteration: 68  train.loss : 0.33108 None train.compute_ips : 695.88 images/s train.total_ips : 695.73 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:35.738270 - Epoch: 1 Iteration: 69  train.loss : 0.32848 None train.compute_ips : 695.66 images/s train.total_ips : 695.49 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:36.108539 - Epoch: 1 Iteration: 70  train.loss : 0.32572 None train.compute_ips : 691.59 images/s train.total_ips : 691.44 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:36.480428 - Epoch: 1 Iteration: 71  train.loss : 0.32269 None train.compute_ips : 688.59 images/s train.total_ips : 688.45 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:36.848926 - Epoch: 1 Iteration: 72  train.loss : 0.31967 None train.compute_ips : 694.93 images/s train.total_ips : 694.76 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:37.218070 - Epoch: 1 Iteration: 73  train.loss : 0.31665 None train.compute_ips : 693.70 images/s train.total_ips : 693.55 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:37.586155 - Epoch: 1 Iteration: 74  train.loss : 0.31359 None train.compute_ips : 695.72 images/s train.total_ips : 695.56 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:37.954728 - Epoch: 1 Iteration: 75  train.loss : 0.31037 None train.compute_ips : 694.79 images/s train.total_ips : 694.63 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:38.323378 - Epoch: 1 Iteration: 76  train.loss : 0.30730 None train.compute_ips : 694.63 images/s train.total_ips : 694.48 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:38.696454 - Epoch: 1 Iteration: 77  train.loss : 0.30457 None train.compute_ips : 686.40 images/s train.total_ips : 686.24 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:39.066464 - Epoch: 1 Iteration: 78  train.loss : 0.30170 None train.compute_ips : 692.10 images/s train.total_ips : 691.95 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:39.434841 - Epoch: 1 Iteration: 79  train.loss : 0.29936 None train.compute_ips : 695.16 images/s train.total_ips : 695.00 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:39.803560 - Epoch: 1 Iteration: 80  train.loss : 0.29733 None train.compute_ips : 694.53 images/s train.total_ips : 694.35 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:40.173146 - Epoch: 1 Iteration: 81  train.loss : 0.29558 None train.compute_ips : 692.88 images/s train.total_ips : 692.72 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:40.540162 - Epoch: 1 Iteration: 82  train.loss : 0.29386 None train.compute_ips : 697.75 images/s train.total_ips : 697.57 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:40.912070 - Epoch: 1 Iteration: 83  train.loss : 0.29244 None train.compute_ips : 688.54 images/s train.total_ips : 688.39 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:41.282662 - Epoch: 1 Iteration: 84  train.loss : 0.29098 None train.compute_ips : 690.99 images/s train.total_ips : 690.84 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:41.654230 - Epoch: 1 Iteration: 85  train.loss : 0.28948 None train.compute_ips : 689.18 images/s train.total_ips : 689.04 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:42.024341 - Epoch: 1 Iteration: 86  train.loss : 0.28786 None train.compute_ips : 691.90 images/s train.total_ips : 691.74 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:42.392477 - Epoch: 1 Iteration: 87  train.loss : 0.28620 None train.compute_ips : 695.62 images/s train.total_ips : 695.46 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:42.759249 - Epoch: 1 Iteration: 88  train.loss : 0.28466 None train.compute_ips : 698.20 images/s train.total_ips : 698.03 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:43.130068 - Epoch: 1 Iteration: 89  train.loss : 0.28323 None train.compute_ips : 690.57 images/s train.total_ips : 690.42 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:43.500162 - Epoch: 1 Iteration: 90  train.loss : 0.28189 None train.compute_ips : 692.00 images/s train.total_ips : 691.77 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:43.872739 - Epoch: 1 Iteration: 91  train.loss : 0.28063 None train.compute_ips : 687.30 images/s train.total_ips : 687.15 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:44.242485 - Epoch: 1 Iteration: 92  train.loss : 0.27947 None train.compute_ips : 692.59 images/s train.total_ips : 692.44 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:44.611635 - Epoch: 1 Iteration: 93  train.loss : 0.27841 None train.compute_ips : 693.71 images/s train.total_ips : 693.55 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:44.979213 - Epoch: 1 Iteration: 94  train.loss : 0.27723 None train.compute_ips : 696.67 images/s train.total_ips : 696.52 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:45.347641 - Epoch: 1 Iteration: 95  train.loss : 0.27609 None train.compute_ips : 695.06 images/s train.total_ips : 694.89 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:45.718461 - Epoch: 1 Iteration: 96  train.loss : 0.27495 None train.compute_ips : 690.57 images/s train.total_ips : 690.42 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:46.089884 - Epoch: 1 Iteration: 97  train.loss : 0.27381 None train.compute_ips : 689.46 images/s train.total_ips : 689.30 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:46.460053 - Epoch: 1 Iteration: 98  train.loss : 0.27262 None train.compute_ips : 691.81 images/s train.total_ips : 691.63 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:46.830607 - Epoch: 1 Iteration: 99  train.loss : 0.27149 None train.compute_ips : 691.07 images/s train.total_ips : 690.91 images/s train.lr : 0.1 
+DLL 2023-07-12 19:53:52.204287 - Epoch: 1  train.loss : 0.39973 None train.compute_ips : 693.86 images/s train.total_ips : 693.71 images/s train.lr : 0.1  val.top1 : None % val.top5 : None % val.loss : None None val.compute_ips : None images/s val.total_ips : None images/s val.compute_latency : None s
+DLL 2023-07-12 19:53:52.204811 - Epoch: 1  train.data_time : 0.00008 s train.compute_time : 0.36896 s val.data_time : None s val.compute_latency_at100 : None s val.compute_latency_at99 : None s val.compute_latency_at95 : None s
+SAVING ./checkpoint_0001.pth.tar
+DLL 2023-07-12 19:53:52.485784 - Summary: train.loss : 4.65508 None train.compute_ips : 689.71 images/s train.total_ips : 689.55 images/s train.lr : 0.1  val.top1 : None % val.top5 : None % val.loss : None None val.compute_ips : None images/s val.total_ips : None images/s val.compute_latency : None s
+DLL 2023-07-12 19:53:52.485940 - Summary: train.data_time : 0.00008 s train.compute_time : 0.38476 s val.data_time : None s val.compute_latency_at100 : None s val.compute_latency_at99 : None s val.compute_latency_at95 : None s
+Experiment ended
+['/opt/conda/bin/python', '-u', './main.py', '/data/imagenet', '--arch', 'resnet50', '--amp', '--static-loss-scale', '256', '--epochs', '2', '--prof', '100', '--batch-size', '256', '--raport-file', 'benchmark.json', '--print-freq', '1', '--training-only', '--data-backend', 'synthetic']
+DONE!
diff --git a/pytorch/results/4090laptop_v1/PyTorch_resnet50_AMP/benchmark.para b/pytorch/results/4090laptop_v1/PyTorch_resnet50_AMP/benchmark.para
new file mode 100755
index 00000000..8df2db16
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_resnet50_AMP/benchmark.para
@@ -0,0 +1,2 @@
+GLOBAL_BATCH 256
+GPU 1
diff --git a/pytorch/results/4090laptop_v1/PyTorch_resnet50_FP32/12-07-2023_20-15-45.txt b/pytorch/results/4090laptop_v1/PyTorch_resnet50_FP32/12-07-2023_20-15-45.txt
new file mode 100755
index 00000000..c26ce12f
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_resnet50_FP32/12-07-2023_20-15-45.txt
@@ -0,0 +1,217 @@
+Namespace(conv_init='fan_in', fused_se=True, last_bn_0_init=False, num_classes=1000, pretrained=False, pretrained_from_file=None, trt=False)
+Training process 0 affinity: {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}
+DLL 2023-07-12 20:15:48.665573 - PARAMETER data : /data/imagenet  data_backend : synthetic  interpolation : bilinear  arch : resnet50  workers : 5  prefetch : 2  epochs : 2  run_epochs : -1  early_stopping_patience : -1  image_size : None  batch_size : 128  optimizer_batch_size : -1  lr : 0.1  lr_schedule : step  end_lr : 0  warmup : 0  label_smoothing : 0.0  mixup : 0.0  optimizer : sgd  momentum : 0.9  weight_decay : 0.0001  bn_weight_decay : False  rmsprop_alpha : 0.9  rmsprop_eps : 0.001  nesterov : False  print_freq : 1  resume : None  static_loss_scale : 1  dynamic_loss_scale : False  prof : 100  amp : False  seed : None  gather_checkpoints : 0  raport_file : benchmark.json  evaluate : False  training_only : True  save_checkpoints : True  jit : no  checkpoint_filename : checkpoint.pth.tar  workspace : ./  memory_format : nchw  use_ema : None  augmentation : None  gpu_affinity : none  topk : 5  distributed : False  local_rank : 0  gpu : 0  world_size : 1 
+DLL 2023-07-12 20:15:48.665685 - PARAMETER model.num_classes : 1000  model.last_bn_0_init : False  model.conv_init : fan_in  model.trt : False  model.fused_se : True  model.pretrained_from_file : None  model.pretrained : False 
+ ! Weight decay NOT applied to BN parameters 
+98
+63
+RUNNING EPOCHS FROM 0 TO 2
+DLL 2023-07-12 20:15:51.843923 - Epoch: 0 Iteration: 1  train.loss : 7.12475 None train.compute_ips : 40.29 images/s train.total_ips : 40.29 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:52.177227 - Epoch: 0 Iteration: 2  train.loss : 5.66098 None train.compute_ips : 384.30 images/s train.total_ips : 384.09 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:52.513684 - Epoch: 0 Iteration: 3  train.loss : 5.08634 None train.compute_ips : 380.58 images/s train.total_ips : 380.46 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:52.852449 - Epoch: 0 Iteration: 4  train.loss : 5.54291 None train.compute_ips : 377.96 images/s train.total_ips : 377.88 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:53.187369 - Epoch: 0 Iteration: 5  train.loss : 6.04220 None train.compute_ips : 382.31 images/s train.total_ips : 382.22 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:53.522860 - Epoch: 0 Iteration: 6  train.loss : 6.47146 None train.compute_ips : 381.65 images/s train.total_ips : 381.57 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:53.862381 - Epoch: 0 Iteration: 7  train.loss : 6.73851 None train.compute_ips : 377.15 images/s train.total_ips : 377.07 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:54.198951 - Epoch: 0 Iteration: 8  train.loss : 6.93237 None train.compute_ips : 380.46 images/s train.total_ips : 380.35 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:54.534267 - Epoch: 0 Iteration: 9  train.loss : 6.59313 None train.compute_ips : 381.86 images/s train.total_ips : 381.77 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:54.872333 - Epoch: 0 Iteration: 10  train.loss : 6.19382 None train.compute_ips : 378.77 images/s train.total_ips : 378.67 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:55.209635 - Epoch: 0 Iteration: 11  train.loss : 6.53738 None train.compute_ips : 379.61 images/s train.total_ips : 379.52 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:55.545139 - Epoch: 0 Iteration: 12  train.loss : 7.17838 None train.compute_ips : 381.64 images/s train.total_ips : 381.56 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:55.879655 - Epoch: 0 Iteration: 13  train.loss : 7.95367 None train.compute_ips : 382.78 images/s train.total_ips : 382.68 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:56.218569 - Epoch: 0 Iteration: 14  train.loss : 7.11615 None train.compute_ips : 377.80 images/s train.total_ips : 377.71 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:56.557142 - Epoch: 0 Iteration: 15  train.loss : 6.12769 None train.compute_ips : 378.18 images/s train.total_ips : 378.09 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:56.891605 - Epoch: 0 Iteration: 16  train.loss : 5.90571 None train.compute_ips : 382.83 images/s train.total_ips : 382.74 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:57.229103 - Epoch: 0 Iteration: 17  train.loss : 5.80743 None train.compute_ips : 379.38 images/s train.total_ips : 379.30 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:57.569262 - Epoch: 0 Iteration: 18  train.loss : 5.72950 None train.compute_ips : 376.43 images/s train.total_ips : 376.34 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:57.903842 - Epoch: 0 Iteration: 19  train.loss : 5.71856 None train.compute_ips : 382.70 images/s train.total_ips : 382.61 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:58.239349 - Epoch: 0 Iteration: 20  train.loss : 5.71436 None train.compute_ips : 381.65 images/s train.total_ips : 381.56 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:58.579080 - Epoch: 0 Iteration: 21  train.loss : 5.68031 None train.compute_ips : 376.90 images/s train.total_ips : 376.80 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:58.915809 - Epoch: 0 Iteration: 22  train.loss : 5.61637 None train.compute_ips : 380.26 images/s train.total_ips : 380.18 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:59.252060 - Epoch: 0 Iteration: 23  train.loss : 5.53582 None train.compute_ips : 380.81 images/s train.total_ips : 380.71 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:59.589765 - Epoch: 0 Iteration: 24  train.loss : 5.44781 None train.compute_ips : 379.16 images/s train.total_ips : 379.07 images/s train.lr : 0.1 
+DLL 2023-07-12 20:15:59.928942 - Epoch: 0 Iteration: 25  train.loss : 5.35455 None train.compute_ips : 377.51 images/s train.total_ips : 377.43 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:00.264832 - Epoch: 0 Iteration: 26  train.loss : 5.25916 None train.compute_ips : 381.22 images/s train.total_ips : 381.12 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:00.600788 - Epoch: 0 Iteration: 27  train.loss : 5.16921 None train.compute_ips : 381.13 images/s train.total_ips : 381.04 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:00.940524 - Epoch: 0 Iteration: 28  train.loss : 5.08773 None train.compute_ips : 376.89 images/s train.total_ips : 376.81 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:01.277609 - Epoch: 0 Iteration: 29  train.loss : 5.01335 None train.compute_ips : 379.87 images/s train.total_ips : 379.78 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:01.613254 - Epoch: 0 Iteration: 30  train.loss : 4.94239 None train.compute_ips : 381.48 images/s train.total_ips : 381.40 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:01.955663 - Epoch: 0 Iteration: 31  train.loss : 4.87038 None train.compute_ips : 373.95 images/s train.total_ips : 373.86 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:02.292998 - Epoch: 0 Iteration: 32  train.loss : 4.79733 None train.compute_ips : 379.58 images/s train.total_ips : 379.48 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:02.628566 - Epoch: 0 Iteration: 33  train.loss : 4.72149 None train.compute_ips : 381.57 images/s train.total_ips : 381.48 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:02.970023 - Epoch: 0 Iteration: 34  train.loss : 4.70195 None train.compute_ips : 374.99 images/s train.total_ips : 374.90 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:03.308763 - Epoch: 0 Iteration: 35  train.loss : 4.63766 None train.compute_ips : 377.99 images/s train.total_ips : 377.90 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:03.645040 - Epoch: 0 Iteration: 36  train.loss : 4.86226 None train.compute_ips : 380.77 images/s train.total_ips : 380.69 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:03.982538 - Epoch: 0 Iteration: 37  train.loss : 4.61551 None train.compute_ips : 379.40 images/s train.total_ips : 379.31 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:04.322401 - Epoch: 0 Iteration: 38  train.loss : 4.48443 None train.compute_ips : 376.77 images/s train.total_ips : 376.66 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:04.660125 - Epoch: 0 Iteration: 39  train.loss : 4.42722 None train.compute_ips : 379.14 images/s train.total_ips : 379.04 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:04.995616 - Epoch: 0 Iteration: 40  train.loss : 4.35420 None train.compute_ips : 381.66 images/s train.total_ips : 381.57 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:05.337314 - Epoch: 0 Iteration: 41  train.loss : 4.26707 None train.compute_ips : 374.72 images/s train.total_ips : 374.63 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:05.675385 - Epoch: 0 Iteration: 42  train.loss : 4.16807 None train.compute_ips : 378.74 images/s train.total_ips : 378.66 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:06.010404 - Epoch: 0 Iteration: 43  train.loss : 4.05970 None train.compute_ips : 382.22 images/s train.total_ips : 382.11 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:06.352687 - Epoch: 0 Iteration: 44  train.loss : 3.98756 None train.compute_ips : 374.09 images/s train.total_ips : 374.00 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:06.690703 - Epoch: 0 Iteration: 45  train.loss : 3.87650 None train.compute_ips : 378.82 images/s train.total_ips : 378.72 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:07.027147 - Epoch: 0 Iteration: 46  train.loss : 3.82399 None train.compute_ips : 380.57 images/s train.total_ips : 380.48 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:07.368499 - Epoch: 0 Iteration: 47  train.loss : 3.89782 None train.compute_ips : 375.10 images/s train.total_ips : 375.02 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:07.707515 - Epoch: 0 Iteration: 48  train.loss : 3.64989 None train.compute_ips : 377.69 images/s train.total_ips : 377.60 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:08.044900 - Epoch: 0 Iteration: 49  train.loss : 3.55207 None train.compute_ips : 379.54 images/s train.total_ips : 379.43 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:08.381699 - Epoch: 0 Iteration: 50  train.loss : 3.41933 None train.compute_ips : 380.17 images/s train.total_ips : 380.09 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:08.721918 - Epoch: 0 Iteration: 51  train.loss : 3.27513 None train.compute_ips : 376.35 images/s train.total_ips : 376.27 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:09.060671 - Epoch: 0 Iteration: 52  train.loss : 3.13365 None train.compute_ips : 377.98 images/s train.total_ips : 377.89 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:09.397783 - Epoch: 0 Iteration: 53  train.loss : 2.98547 None train.compute_ips : 379.82 images/s train.total_ips : 379.73 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:09.738773 - Epoch: 0 Iteration: 54  train.loss : 2.83461 None train.compute_ips : 375.50 images/s train.total_ips : 375.42 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:10.076725 - Epoch: 0 Iteration: 55  train.loss : 2.68688 None train.compute_ips : 378.88 images/s train.total_ips : 378.79 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:10.412925 - Epoch: 0 Iteration: 56  train.loss : 2.54085 None train.compute_ips : 380.86 images/s train.total_ips : 380.77 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:10.754982 - Epoch: 0 Iteration: 57  train.loss : 2.39503 None train.compute_ips : 374.33 images/s train.total_ips : 374.24 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:11.094638 - Epoch: 0 Iteration: 58  train.loss : 2.25077 None train.compute_ips : 376.98 images/s train.total_ips : 376.89 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:11.432419 - Epoch: 0 Iteration: 59  train.loss : 2.10841 None train.compute_ips : 379.07 images/s train.total_ips : 378.98 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:11.770494 - Epoch: 0 Iteration: 60  train.loss : 1.96764 None train.compute_ips : 378.75 images/s train.total_ips : 378.66 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:12.111070 - Epoch: 0 Iteration: 61  train.loss : 1.82986 None train.compute_ips : 375.97 images/s train.total_ips : 375.87 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:12.450741 - Epoch: 0 Iteration: 62  train.loss : 1.68947 None train.compute_ips : 376.96 images/s train.total_ips : 376.87 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:12.786600 - Epoch: 0 Iteration: 63  train.loss : 1.55378 None train.compute_ips : 381.24 images/s train.total_ips : 381.15 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:13.126583 - Epoch: 0 Iteration: 64  train.loss : 1.42267 None train.compute_ips : 376.62 images/s train.total_ips : 376.53 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:13.466458 - Epoch: 0 Iteration: 65  train.loss : 1.29397 None train.compute_ips : 376.73 images/s train.total_ips : 376.65 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:13.803082 - Epoch: 0 Iteration: 66  train.loss : 1.16959 None train.compute_ips : 380.37 images/s train.total_ips : 380.29 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:14.142942 - Epoch: 0 Iteration: 67  train.loss : 1.05036 None train.compute_ips : 376.76 images/s train.total_ips : 376.68 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:14.481983 - Epoch: 0 Iteration: 68  train.loss : 0.93560 None train.compute_ips : 377.66 images/s train.total_ips : 377.57 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:14.819355 - Epoch: 0 Iteration: 69  train.loss : 0.82433 None train.compute_ips : 379.53 images/s train.total_ips : 379.45 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:15.158769 - Epoch: 0 Iteration: 70  train.loss : 0.71803 None train.compute_ips : 377.25 images/s train.total_ips : 377.16 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:15.497552 - Epoch: 0 Iteration: 71  train.loss : 0.61860 None train.compute_ips : 377.95 images/s train.total_ips : 377.86 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:15.837311 - Epoch: 0 Iteration: 72  train.loss : 0.53095 None train.compute_ips : 376.87 images/s train.total_ips : 376.78 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:16.175549 - Epoch: 0 Iteration: 73  train.loss : 0.45684 None train.compute_ips : 378.56 images/s train.total_ips : 378.47 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:16.514527 - Epoch: 0 Iteration: 74  train.loss : 0.39373 None train.compute_ips : 377.73 images/s train.total_ips : 377.65 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:16.855832 - Epoch: 0 Iteration: 75  train.loss : 0.34086 None train.compute_ips : 375.16 images/s train.total_ips : 375.07 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:17.193328 - Epoch: 0 Iteration: 76  train.loss : 0.30108 None train.compute_ips : 379.39 images/s train.total_ips : 379.31 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:17.533939 - Epoch: 0 Iteration: 77  train.loss : 0.27223 None train.compute_ips : 375.92 images/s train.total_ips : 375.83 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:17.873335 - Epoch: 0 Iteration: 78  train.loss : 0.24642 None train.compute_ips : 377.27 images/s train.total_ips : 377.18 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:18.210817 - Epoch: 0 Iteration: 79  train.loss : 0.22794 None train.compute_ips : 379.41 images/s train.total_ips : 379.32 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:18.552676 - Epoch: 0 Iteration: 80  train.loss : 0.19943 None train.compute_ips : 374.56 images/s train.total_ips : 374.46 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:18.891212 - Epoch: 0 Iteration: 81  train.loss : 0.18561 None train.compute_ips : 378.23 images/s train.total_ips : 378.14 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:19.230489 - Epoch: 0 Iteration: 82  train.loss : 0.16790 None train.compute_ips : 377.40 images/s train.total_ips : 377.32 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:19.570020 - Epoch: 0 Iteration: 83  train.loss : 0.15262 None train.compute_ips : 377.14 images/s train.total_ips : 377.03 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:19.909591 - Epoch: 0 Iteration: 84  train.loss : 0.14375 None train.compute_ips : 377.08 images/s train.total_ips : 376.99 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:20.248808 - Epoch: 0 Iteration: 85  train.loss : 0.13493 None train.compute_ips : 377.48 images/s train.total_ips : 377.37 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:20.586708 - Epoch: 0 Iteration: 86  train.loss : 0.12707 None train.compute_ips : 378.93 images/s train.total_ips : 378.85 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:20.928417 - Epoch: 0 Iteration: 87  train.loss : 0.12123 None train.compute_ips : 374.72 images/s train.total_ips : 374.63 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:21.268270 - Epoch: 0 Iteration: 88  train.loss : 0.11374 None train.compute_ips : 376.76 images/s train.total_ips : 376.67 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:21.604804 - Epoch: 0 Iteration: 89  train.loss : 0.10792 None train.compute_ips : 380.47 images/s train.total_ips : 380.39 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:21.947284 - Epoch: 0 Iteration: 90  train.loss : 0.10159 None train.compute_ips : 373.88 images/s train.total_ips : 373.79 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:22.287547 - Epoch: 0 Iteration: 91  train.loss : 0.09729 None train.compute_ips : 376.35 images/s train.total_ips : 376.25 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:22.625576 - Epoch: 0 Iteration: 92  train.loss : 0.09423 None train.compute_ips : 378.80 images/s train.total_ips : 378.69 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:22.965841 - Epoch: 0 Iteration: 93  train.loss : 0.09129 None train.compute_ips : 376.30 images/s train.total_ips : 376.22 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:23.306609 - Epoch: 0 Iteration: 94  train.loss : 0.08932 None train.compute_ips : 375.74 images/s train.total_ips : 375.66 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:23.645557 - Epoch: 0 Iteration: 95  train.loss : 0.08775 None train.compute_ips : 377.77 images/s train.total_ips : 377.68 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:23.982840 - Epoch: 0 Iteration: 96  train.loss : 0.08379 None train.compute_ips : 379.63 images/s train.total_ips : 379.54 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:24.324194 - Epoch: 0 Iteration: 97  train.loss : 0.08195 None train.compute_ips : 375.10 images/s train.total_ips : 375.01 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:24.663864 - Epoch: 0 Iteration: 98  train.loss : 0.08123 None train.compute_ips : 376.96 images/s train.total_ips : 376.87 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:25.000855 - Epoch: 0 Iteration: 99  train.loss : 0.07945 None train.compute_ips : 379.96 images/s train.total_ips : 379.87 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:30.348287 - Epoch: 0  train.loss : 3.08932 None train.compute_ips : 375.05 images/s train.total_ips : 374.96 images/s train.lr : 0.1  val.top1 : None % val.top5 : None % val.loss : None None val.compute_ips : None images/s val.total_ips : None images/s val.compute_latency : None s
+DLL 2023-07-12 20:16:30.348628 - Epoch: 0  train.data_time : 0.00008 s train.compute_time : 0.36664 s val.data_time : None s val.compute_latency_at100 : None s val.compute_latency_at99 : None s val.compute_latency_at95 : None s
+SAVING ./checkpoint_0000.pth.tar
+DLL 2023-07-12 20:16:31.016944 - Epoch: 1 Iteration: 1  train.loss : 0.07650 None train.compute_ips : 384.14 images/s train.total_ips : 384.11 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:31.353721 - Epoch: 1 Iteration: 2  train.loss : 0.07522 None train.compute_ips : 380.27 images/s train.total_ips : 380.11 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:31.688920 - Epoch: 1 Iteration: 3  train.loss : 0.07403 None train.compute_ips : 382.00 images/s train.total_ips : 381.90 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:32.030898 - Epoch: 1 Iteration: 4  train.loss : 0.07275 None train.compute_ips : 374.42 images/s train.total_ips : 374.33 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:32.368191 - Epoch: 1 Iteration: 5  train.loss : 0.07239 None train.compute_ips : 379.62 images/s train.total_ips : 379.53 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:32.704023 - Epoch: 1 Iteration: 6  train.loss : 0.07147 None train.compute_ips : 381.27 images/s train.total_ips : 381.18 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:33.046046 - Epoch: 1 Iteration: 7  train.loss : 0.07023 None train.compute_ips : 374.37 images/s train.total_ips : 374.28 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:33.382088 - Epoch: 1 Iteration: 8  train.loss : 0.06941 None train.compute_ips : 381.03 images/s train.total_ips : 380.94 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:33.718806 - Epoch: 1 Iteration: 9  train.loss : 0.06855 None train.compute_ips : 380.28 images/s train.total_ips : 380.19 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:34.058884 - Epoch: 1 Iteration: 10  train.loss : 0.06782 None train.compute_ips : 376.53 images/s train.total_ips : 376.43 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:34.396961 - Epoch: 1 Iteration: 11  train.loss : 0.06715 None train.compute_ips : 378.75 images/s train.total_ips : 378.65 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:34.734504 - Epoch: 1 Iteration: 12  train.loss : 0.06594 None train.compute_ips : 379.35 images/s train.total_ips : 379.26 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:35.072324 - Epoch: 1 Iteration: 13  train.loss : 0.06507 None train.compute_ips : 379.04 images/s train.total_ips : 378.94 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:35.410195 - Epoch: 1 Iteration: 14  train.loss : 0.06489 None train.compute_ips : 378.97 images/s train.total_ips : 378.88 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:35.750341 - Epoch: 1 Iteration: 15  train.loss : 0.06399 None train.compute_ips : 376.44 images/s train.total_ips : 376.35 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:36.087222 - Epoch: 1 Iteration: 16  train.loss : 0.06360 None train.compute_ips : 380.08 images/s train.total_ips : 379.99 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:36.426703 - Epoch: 1 Iteration: 17  train.loss : 0.06300 None train.compute_ips : 377.17 images/s train.total_ips : 377.09 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:36.765901 - Epoch: 1 Iteration: 18  train.loss : 0.06221 None train.compute_ips : 377.49 images/s train.total_ips : 377.40 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:37.102998 - Epoch: 1 Iteration: 19  train.loss : 0.06152 None train.compute_ips : 379.85 images/s train.total_ips : 379.75 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:37.443116 - Epoch: 1 Iteration: 20  train.loss : 0.06091 None train.compute_ips : 376.46 images/s train.total_ips : 376.38 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:37.781611 - Epoch: 1 Iteration: 21  train.loss : 0.06055 None train.compute_ips : 378.27 images/s train.total_ips : 378.18 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:38.121099 - Epoch: 1 Iteration: 22  train.loss : 0.06007 None train.compute_ips : 377.16 images/s train.total_ips : 377.07 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:38.459517 - Epoch: 1 Iteration: 23  train.loss : 0.05924 None train.compute_ips : 378.35 images/s train.total_ips : 378.27 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:38.797602 - Epoch: 1 Iteration: 24  train.loss : 0.05883 None train.compute_ips : 378.73 images/s train.total_ips : 378.64 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:39.137963 - Epoch: 1 Iteration: 25  train.loss : 0.05838 None train.compute_ips : 376.21 images/s train.total_ips : 376.11 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:39.474742 - Epoch: 1 Iteration: 26  train.loss : 0.05763 None train.compute_ips : 380.20 images/s train.total_ips : 380.11 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:39.813610 - Epoch: 1 Iteration: 27  train.loss : 0.05718 None train.compute_ips : 377.86 images/s train.total_ips : 377.77 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:40.154063 - Epoch: 1 Iteration: 28  train.loss : 0.05643 None train.compute_ips : 376.10 images/s train.total_ips : 376.01 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:40.491845 - Epoch: 1 Iteration: 29  train.loss : 0.05567 None train.compute_ips : 379.07 images/s train.total_ips : 378.98 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:40.829701 - Epoch: 1 Iteration: 30  train.loss : 0.05418 None train.compute_ips : 378.99 images/s train.total_ips : 378.90 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:41.169477 - Epoch: 1 Iteration: 31  train.loss : 0.05258 None train.compute_ips : 376.86 images/s train.total_ips : 376.77 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:41.509214 - Epoch: 1 Iteration: 32  train.loss : 0.05251 None train.compute_ips : 376.91 images/s train.total_ips : 376.80 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:41.848421 - Epoch: 1 Iteration: 33  train.loss : 0.05133 None train.compute_ips : 377.49 images/s train.total_ips : 377.39 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:42.186774 - Epoch: 1 Iteration: 34  train.loss : 0.05025 None train.compute_ips : 378.44 images/s train.total_ips : 378.34 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:42.527966 - Epoch: 1 Iteration: 35  train.loss : 0.05048 None train.compute_ips : 375.28 images/s train.total_ips : 375.19 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:42.866462 - Epoch: 1 Iteration: 36  train.loss : 0.04965 None train.compute_ips : 378.27 images/s train.total_ips : 378.18 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:43.203971 - Epoch: 1 Iteration: 37  train.loss : 0.04823 None train.compute_ips : 379.38 images/s train.total_ips : 379.29 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:43.546576 - Epoch: 1 Iteration: 38  train.loss : 0.04755 None train.compute_ips : 373.74 images/s train.total_ips : 373.65 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:43.884805 - Epoch: 1 Iteration: 39  train.loss : 0.04718 None train.compute_ips : 378.57 images/s train.total_ips : 378.48 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:44.222879 - Epoch: 1 Iteration: 40  train.loss : 0.04622 None train.compute_ips : 378.75 images/s train.total_ips : 378.66 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:44.562670 - Epoch: 1 Iteration: 41  train.loss : 0.04617 None train.compute_ips : 376.83 images/s train.total_ips : 376.74 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:44.902082 - Epoch: 1 Iteration: 42  train.loss : 0.04542 None train.compute_ips : 377.27 images/s train.total_ips : 377.16 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:45.241652 - Epoch: 1 Iteration: 43  train.loss : 0.04429 None train.compute_ips : 377.08 images/s train.total_ips : 376.99 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:45.578715 - Epoch: 1 Iteration: 44  train.loss : 0.04354 None train.compute_ips : 379.88 images/s train.total_ips : 379.79 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:45.919836 - Epoch: 1 Iteration: 45  train.loss : 0.04302 None train.compute_ips : 375.38 images/s train.total_ips : 375.29 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:46.258879 - Epoch: 1 Iteration: 46  train.loss : 0.04219 None train.compute_ips : 377.69 images/s train.total_ips : 377.58 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:46.595357 - Epoch: 1 Iteration: 47  train.loss : 0.04192 None train.compute_ips : 380.56 images/s train.total_ips : 380.44 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:46.937190 - Epoch: 1 Iteration: 48  train.loss : 0.04116 None train.compute_ips : 374.58 images/s train.total_ips : 374.49 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:47.276294 - Epoch: 1 Iteration: 49  train.loss : 0.04025 None train.compute_ips : 377.60 images/s train.total_ips : 377.51 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:47.613346 - Epoch: 1 Iteration: 50  train.loss : 0.04020 None train.compute_ips : 379.92 images/s train.total_ips : 379.80 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:47.954559 - Epoch: 1 Iteration: 51  train.loss : 0.03906 None train.compute_ips : 375.27 images/s train.total_ips : 375.17 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:48.293509 - Epoch: 1 Iteration: 52  train.loss : 0.03884 None train.compute_ips : 377.79 images/s train.total_ips : 377.69 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:48.632399 - Epoch: 1 Iteration: 53  train.loss : 0.03828 None train.compute_ips : 377.83 images/s train.total_ips : 377.73 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:48.969904 - Epoch: 1 Iteration: 54  train.loss : 0.03759 None train.compute_ips : 379.39 images/s train.total_ips : 379.30 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:49.310069 - Epoch: 1 Iteration: 55  train.loss : 0.03712 None train.compute_ips : 376.41 images/s train.total_ips : 376.32 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:49.650597 - Epoch: 1 Iteration: 56  train.loss : 0.03671 None train.compute_ips : 376.01 images/s train.total_ips : 375.93 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:49.987767 - Epoch: 1 Iteration: 57  train.loss : 0.03603 None train.compute_ips : 379.76 images/s train.total_ips : 379.67 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:50.328415 - Epoch: 1 Iteration: 58  train.loss : 0.03563 None train.compute_ips : 375.89 images/s train.total_ips : 375.80 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:50.669499 - Epoch: 1 Iteration: 59  train.loss : 0.03521 None train.compute_ips : 375.40 images/s train.total_ips : 375.31 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:51.006754 - Epoch: 1 Iteration: 60  train.loss : 0.03472 None train.compute_ips : 379.67 images/s train.total_ips : 379.58 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:51.347487 - Epoch: 1 Iteration: 61  train.loss : 0.03426 None train.compute_ips : 375.79 images/s train.total_ips : 375.70 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:51.688158 - Epoch: 1 Iteration: 62  train.loss : 0.03395 None train.compute_ips : 375.86 images/s train.total_ips : 375.77 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:52.027678 - Epoch: 1 Iteration: 63  train.loss : 0.03353 None train.compute_ips : 377.14 images/s train.total_ips : 377.05 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:52.366312 - Epoch: 1 Iteration: 64  train.loss : 0.03311 None train.compute_ips : 378.12 images/s train.total_ips : 378.03 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:52.706666 - Epoch: 1 Iteration: 65  train.loss : 0.03278 None train.compute_ips : 376.21 images/s train.total_ips : 376.13 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:53.047166 - Epoch: 1 Iteration: 66  train.loss : 0.03245 None train.compute_ips : 376.06 images/s train.total_ips : 375.95 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:53.384926 - Epoch: 1 Iteration: 67  train.loss : 0.03211 None train.compute_ips : 379.09 images/s train.total_ips : 379.01 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:53.726166 - Epoch: 1 Iteration: 68  train.loss : 0.03176 None train.compute_ips : 375.23 images/s train.total_ips : 375.14 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:54.066691 - Epoch: 1 Iteration: 69  train.loss : 0.03149 None train.compute_ips : 376.04 images/s train.total_ips : 375.95 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:54.404428 - Epoch: 1 Iteration: 70  train.loss : 0.03120 None train.compute_ips : 379.12 images/s train.total_ips : 379.03 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:54.746098 - Epoch: 1 Iteration: 71  train.loss : 0.03090 None train.compute_ips : 374.76 images/s train.total_ips : 374.67 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:55.086244 - Epoch: 1 Iteration: 72  train.loss : 0.03062 None train.compute_ips : 376.44 images/s train.total_ips : 376.35 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:55.425167 - Epoch: 1 Iteration: 73  train.loss : 0.03037 None train.compute_ips : 377.79 images/s train.total_ips : 377.70 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:55.765175 - Epoch: 1 Iteration: 74  train.loss : 0.03011 None train.compute_ips : 376.59 images/s train.total_ips : 376.50 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:56.106559 - Epoch: 1 Iteration: 75  train.loss : 0.02986 None train.compute_ips : 375.07 images/s train.total_ips : 374.98 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:56.446513 - Epoch: 1 Iteration: 76  train.loss : 0.02962 None train.compute_ips : 376.66 images/s train.total_ips : 376.57 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:56.783517 - Epoch: 1 Iteration: 77  train.loss : 0.02941 None train.compute_ips : 379.97 images/s train.total_ips : 379.86 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:57.125172 - Epoch: 1 Iteration: 78  train.loss : 0.02919 None train.compute_ips : 374.77 images/s train.total_ips : 374.68 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:57.465632 - Epoch: 1 Iteration: 79  train.loss : 0.02897 None train.compute_ips : 376.09 images/s train.total_ips : 376.01 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:57.803638 - Epoch: 1 Iteration: 80  train.loss : 0.02878 None train.compute_ips : 378.86 images/s train.total_ips : 378.77 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:58.144681 - Epoch: 1 Iteration: 81  train.loss : 0.02859 None train.compute_ips : 375.49 images/s train.total_ips : 375.37 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:58.484574 - Epoch: 1 Iteration: 82  train.loss : 0.02841 None train.compute_ips : 376.75 images/s train.total_ips : 376.63 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:58.824721 - Epoch: 1 Iteration: 83  train.loss : 0.02823 None train.compute_ips : 376.44 images/s train.total_ips : 376.35 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:59.163682 - Epoch: 1 Iteration: 84  train.loss : 0.02808 None train.compute_ips : 377.76 images/s train.total_ips : 377.67 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:59.504280 - Epoch: 1 Iteration: 85  train.loss : 0.02791 None train.compute_ips : 375.95 images/s train.total_ips : 375.85 images/s train.lr : 0.1 
+DLL 2023-07-12 20:16:59.844882 - Epoch: 1 Iteration: 86  train.loss : 0.02776 None train.compute_ips : 375.93 images/s train.total_ips : 375.84 images/s train.lr : 0.1 
+DLL 2023-07-12 20:17:00.183453 - Epoch: 1 Iteration: 87  train.loss : 0.02761 None train.compute_ips : 378.19 images/s train.total_ips : 378.10 images/s train.lr : 0.1 
+DLL 2023-07-12 20:17:00.524454 - Epoch: 1 Iteration: 88  train.loss : 0.02747 None train.compute_ips : 375.50 images/s train.total_ips : 375.41 images/s train.lr : 0.1 
+DLL 2023-07-12 20:17:00.864834 - Epoch: 1 Iteration: 89  train.loss : 0.02733 None train.compute_ips : 376.19 images/s train.total_ips : 376.09 images/s train.lr : 0.1 
+DLL 2023-07-12 20:17:01.203338 - Epoch: 1 Iteration: 90  train.loss : 0.02719 None train.compute_ips : 378.28 images/s train.total_ips : 378.18 images/s train.lr : 0.1 
+DLL 2023-07-12 20:17:01.544073 - Epoch: 1 Iteration: 91  train.loss : 0.02707 None train.compute_ips : 375.79 images/s train.total_ips : 375.70 images/s train.lr : 0.1 
+DLL 2023-07-12 20:17:01.884707 - Epoch: 1 Iteration: 92  train.loss : 0.02694 None train.compute_ips : 375.91 images/s train.total_ips : 375.82 images/s train.lr : 0.1 
+DLL 2023-07-12 20:17:02.224512 - Epoch: 1 Iteration: 93  train.loss : 0.02682 None train.compute_ips : 376.84 images/s train.total_ips : 376.73 images/s train.lr : 0.1 
+DLL 2023-07-12 20:17:02.563915 - Epoch: 1 Iteration: 94  train.loss : 0.02670 None train.compute_ips : 377.27 images/s train.total_ips : 377.17 images/s train.lr : 0.1 
+DLL 2023-07-12 20:17:02.905844 - Epoch: 1 Iteration: 95  train.loss : 0.02659 None train.compute_ips : 374.47 images/s train.total_ips : 374.39 images/s train.lr : 0.1 
+DLL 2023-07-12 20:17:03.246393 - Epoch: 1 Iteration: 96  train.loss : 0.02648 None train.compute_ips : 375.99 images/s train.total_ips : 375.90 images/s train.lr : 0.1 
+DLL 2023-07-12 20:17:03.582409 - Epoch: 1 Iteration: 97  train.loss : 0.02637 None train.compute_ips : 381.07 images/s train.total_ips : 380.97 images/s train.lr : 0.1 
+DLL 2023-07-12 20:17:03.925555 - Epoch: 1 Iteration: 98  train.loss : 0.02626 None train.compute_ips : 373.15 images/s train.total_ips : 373.06 images/s train.lr : 0.1 
+DLL 2023-07-12 20:17:04.266277 - Epoch: 1 Iteration: 99  train.loss : 0.02616 None train.compute_ips : 375.80 images/s train.total_ips : 375.71 images/s train.lr : 0.1 
+DLL 2023-07-12 20:17:09.609526 - Epoch: 1  train.loss : 0.04397 None train.compute_ips : 377.50 images/s train.total_ips : 377.41 images/s train.lr : 0.1  val.top1 : None % val.top5 : None % val.loss : None None val.compute_ips : None images/s val.total_ips : None images/s val.compute_latency : None s
+DLL 2023-07-12 20:17:09.609913 - Epoch: 1  train.data_time : 0.00008 s train.compute_time : 0.33908 s val.data_time : None s val.compute_latency_at100 : None s val.compute_latency_at99 : None s val.compute_latency_at95 : None s
+SAVING ./checkpoint_0001.pth.tar
+DLL 2023-07-12 20:17:09.970037 - Summary: train.loss : 3.08932 None train.compute_ips : 376.28 images/s train.total_ips : 376.18 images/s train.lr : 0.1  val.top1 : None % val.top5 : None % val.loss : None None val.compute_ips : None images/s val.total_ips : None images/s val.compute_latency : None s
+DLL 2023-07-12 20:17:09.970168 - Summary: train.data_time : 0.00008 s train.compute_time : 0.35286 s val.data_time : None s val.compute_latency_at100 : None s val.compute_latency_at99 : None s val.compute_latency_at95 : None s
+Experiment ended
+['/opt/conda/bin/python', '-u', './main.py', '/data/imagenet', '--arch', 'resnet50', '--epochs', '2', '--prof', '100', '--batch-size', '128', '--raport-file', 'benchmark.json', '--print-freq', '1', '--training-only', '--data-backend', 'synthetic']
+DONE!
diff --git a/pytorch/results/4090laptop_v1/PyTorch_resnet50_FP32/benchmark.para b/pytorch/results/4090laptop_v1/PyTorch_resnet50_FP32/benchmark.para
new file mode 100755
index 00000000..497e227d
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_resnet50_FP32/benchmark.para
@@ -0,0 +1,2 @@
+GLOBAL_BATCH 128
+GPU 1
diff --git a/pytorch/results/4090laptop_v1/PyTorch_tacotron2_FP16/12-07-2023_19-51-19.txt b/pytorch/results/4090laptop_v1/PyTorch_tacotron2_FP16/12-07-2023_19-51-19.txt
new file mode 100755
index 00000000..935e6ea3
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_tacotron2_FP16/12-07-2023_19-51-19.txt
@@ -0,0 +1,173 @@
+OMP: Info #276: omp_set_nested routine deprecated, please use omp_set_max_active_levels instead.
+DLL 2023-07-12 19:51:30.620554 - PARAMETER output : ./ 
+DLL 2023-07-12 19:51:30.620584 - PARAMETER dataset_path : /data/tacotron2/LJSpeech-1.1 
+DLL 2023-07-12 19:51:30.620596 - PARAMETER model_name : Tacotron2 
+DLL 2023-07-12 19:51:30.620604 - PARAMETER log_file : nvlog.json 
+DLL 2023-07-12 19:51:30.620611 - PARAMETER anneal_steps : None 
+DLL 2023-07-12 19:51:30.620617 - PARAMETER anneal_factor : 0.1 
+DLL 2023-07-12 19:51:30.620625 - PARAMETER config_file : None 
+DLL 2023-07-12 19:51:30.620631 - PARAMETER seed : None 
+DLL 2023-07-12 19:51:30.620637 - PARAMETER epochs : 2 
+DLL 2023-07-12 19:51:30.620644 - PARAMETER epochs_per_checkpoint : 50 
+DLL 2023-07-12 19:51:30.620650 - PARAMETER checkpoint_path :  
+DLL 2023-07-12 19:51:30.620656 - PARAMETER resume_from_last : False 
+DLL 2023-07-12 19:51:30.620663 - PARAMETER dynamic_loss_scaling : True 
+DLL 2023-07-12 19:51:30.620669 - PARAMETER amp : False 
+DLL 2023-07-12 19:51:30.620676 - PARAMETER cudnn_enabled : True 
+DLL 2023-07-12 19:51:30.620682 - PARAMETER cudnn_benchmark : False 
+DLL 2023-07-12 19:51:30.620688 - PARAMETER disable_uniform_initialize_bn_weight : False 
+DLL 2023-07-12 19:51:30.620694 - PARAMETER use_saved_learning_rate : False 
+DLL 2023-07-12 19:51:30.620700 - PARAMETER learning_rate : 0.0 
+DLL 2023-07-12 19:51:30.620706 - PARAMETER weight_decay : 1e-06 
+DLL 2023-07-12 19:51:30.620713 - PARAMETER grad_clip_thresh : 1.0 
+DLL 2023-07-12 19:51:30.620719 - PARAMETER batch_size : 48 
+DLL 2023-07-12 19:51:30.620725 - PARAMETER grad_clip : 5.0 
+DLL 2023-07-12 19:51:30.620732 - PARAMETER load_mel_from_disk : False 
+DLL 2023-07-12 19:51:30.620737 - PARAMETER training_files : filelists/ljs_audio_text_train_subset_625_filelist.txt 
+DLL 2023-07-12 19:51:30.620744 - PARAMETER validation_files : filelists/ljs_audio_text_val_filelist.txt 
+DLL 2023-07-12 19:51:30.620750 - PARAMETER text_cleaners : ['english_cleaners'] 
+DLL 2023-07-12 19:51:30.620758 - PARAMETER max_wav_value : 32768.0 
+DLL 2023-07-12 19:51:30.620764 - PARAMETER sampling_rate : 22050 
+DLL 2023-07-12 19:51:30.620770 - PARAMETER filter_length : 1024 
+DLL 2023-07-12 19:51:30.620776 - PARAMETER hop_length : 256 
+DLL 2023-07-12 19:51:30.620782 - PARAMETER win_length : 1024 
+DLL 2023-07-12 19:51:30.620788 - PARAMETER mel_fmin : 0.0 
+DLL 2023-07-12 19:51:30.620794 - PARAMETER mel_fmax : 8000.0 
+DLL 2023-07-12 19:51:30.620800 - PARAMETER rank : 0 
+DLL 2023-07-12 19:51:30.620806 - PARAMETER world_size : 1 
+DLL 2023-07-12 19:51:30.620812 - PARAMETER dist_url : tcp://localhost:23456 
+DLL 2023-07-12 19:51:30.620818 - PARAMETER group_name : group_name 
+DLL 2023-07-12 19:51:30.620824 - PARAMETER dist_backend : nccl 
+DLL 2023-07-12 19:51:30.620830 - PARAMETER bench_class :  
+DLL 2023-07-12 19:51:30.620836 - PARAMETER model_name : Tacotron2_PyT 
+/workspace/benchmark/SpeechSynthesis/Tacotron2/tacotron2/text/__init__.py:74: SyntaxWarning: "is not" with a literal. Did you mean "!="?
+  return s in _symbol_to_id and s is not '_' and s is not '~'
+/workspace/benchmark/SpeechSynthesis/Tacotron2/tacotron2/text/__init__.py:74: SyntaxWarning: "is not" with a literal. Did you mean "!="?
+  return s in _symbol_to_id and s is not '_' and s is not '~'
+DLL 2023-07-12 19:51:33.979817 - (0, 0) glob_iter/iters_per_epoch : 0/13 
+DLL 2023-07-12 19:51:34.823309 - (0, 0) train_loss : 47.845130920410156 
+DLL 2023-07-12 19:51:36.015227 - (0, 0) train_items_per_sec : 13332.076874625896 items/s
+DLL 2023-07-12 19:51:36.015272 - (0, 0) train_iter_time : 2.0354668109998784 
+DLL 2023-07-12 19:51:36.019404 - (0, 1) glob_iter/iters_per_epoch : 1/13 
+DLL 2023-07-12 19:51:36.307880 - (0, 1) train_loss : 51.652252197265625 
+DLL 2023-07-12 19:51:37.398243 - (0, 1) train_items_per_sec : 20881.337506876593 items/s
+DLL 2023-07-12 19:51:37.398289 - (0, 1) train_iter_time : 1.3788388789998862 
+DLL 2023-07-12 19:51:37.399084 - (0, 2) glob_iter/iters_per_epoch : 2/13 
+DLL 2023-07-12 19:51:37.704416 - (0, 2) train_loss : 46.36100769042969 
+DLL 2023-07-12 19:51:38.855158 - (0, 2) train_items_per_sec : 18514.13454109095 items/s
+DLL 2023-07-12 19:51:38.855206 - (0, 2) train_iter_time : 1.45607670399977 
+DLL 2023-07-12 19:51:38.856201 - (0, 3) glob_iter/iters_per_epoch : 3/13 
+DLL 2023-07-12 19:51:39.184179 - (0, 3) train_loss : 47.39357376098633 
+DLL 2023-07-12 19:51:40.307450 - (0, 3) train_items_per_sec : 18338.652479672408 items/s
+DLL 2023-07-12 19:51:40.307495 - (0, 3) train_iter_time : 1.4512516680001681 
+DLL 2023-07-12 19:51:40.308362 - (0, 4) glob_iter/iters_per_epoch : 4/13 
+DLL 2023-07-12 19:51:40.614632 - (0, 4) train_loss : 43.8939323425293 
+DLL 2023-07-12 19:51:41.755852 - (0, 4) train_items_per_sec : 17491.596562123024 items/s
+DLL 2023-07-12 19:51:41.755904 - (0, 4) train_iter_time : 1.4474950819999322 
+DLL 2023-07-12 19:51:41.756756 - (0, 5) glob_iter/iters_per_epoch : 5/13 
+DLL 2023-07-12 19:51:42.063721 - (0, 5) train_loss : 46.65218734741211 
+DLL 2023-07-12 19:51:43.214680 - (0, 5) train_items_per_sec : 18513.258441063605 items/s
+DLL 2023-07-12 19:51:43.214728 - (0, 5) train_iter_time : 1.4579281159999482 
+DLL 2023-07-12 19:51:43.219383 - (0, 6) glob_iter/iters_per_epoch : 6/13 
+DLL 2023-07-12 19:51:43.507532 - (0, 6) train_loss : 51.1827507019043 
+DLL 2023-07-12 19:51:44.595675 - (0, 6) train_items_per_sec : 20808.07643884989 items/s
+DLL 2023-07-12 19:51:44.595723 - (0, 6) train_iter_time : 1.3762925220003126 
+DLL 2023-07-12 19:51:44.596565 - (0, 7) glob_iter/iters_per_epoch : 7/13 
+DLL 2023-07-12 19:51:44.947041 - (0, 7) train_loss : 49.12106704711914 
+DLL 2023-07-12 19:51:46.120721 - (0, 7) train_items_per_sec : 18777.565660755005 items/s
+DLL 2023-07-12 19:51:46.120769 - (0, 7) train_iter_time : 1.524159229000361 
+DLL 2023-07-12 19:51:46.125909 - (0, 8) glob_iter/iters_per_epoch : 8/13 
+DLL 2023-07-12 19:51:46.426288 - (0, 8) train_loss : 45.45895004272461 
+DLL 2023-07-12 19:51:47.579872 - (0, 8) train_items_per_sec : 18036.160355663626 items/s
+DLL 2023-07-12 19:51:47.579921 - (0, 8) train_iter_time : 1.4539679999998043 
+DLL 2023-07-12 19:51:47.583696 - (0, 9) glob_iter/iters_per_epoch : 9/13 
+DLL 2023-07-12 19:51:47.880982 - (0, 9) train_loss : 45.34659194946289 
+DLL 2023-07-12 19:51:49.000432 - (0, 9) train_items_per_sec : 18370.349341384714 items/s
+DLL 2023-07-12 19:51:49.000483 - (0, 9) train_iter_time : 1.416739525000139 
+DLL 2023-07-12 19:51:49.001319 - (0, 10) glob_iter/iters_per_epoch : 10/13 
+DLL 2023-07-12 19:51:49.320802 - (0, 10) train_loss : 45.51616287231445 
+DLL 2023-07-12 19:51:50.450116 - (0, 10) train_items_per_sec : 18188.168909282143 items/s
+DLL 2023-07-12 19:51:50.450165 - (0, 10) train_iter_time : 1.4487989489998654 
+DLL 2023-07-12 19:51:50.454499 - (0, 11) glob_iter/iters_per_epoch : 11/13 
+DLL 2023-07-12 19:51:50.750555 - (0, 11) train_loss : 48.318519592285156 
+DLL 2023-07-12 19:51:51.871903 - (0, 11) train_items_per_sec : 19297.88134435469 items/s
+DLL 2023-07-12 19:51:51.871951 - (0, 11) train_iter_time : 1.4174094820000391 
+DLL 2023-07-12 19:51:51.872767 - (0, 12) glob_iter/iters_per_epoch : 12/13 
+DLL 2023-07-12 19:51:52.174517 - (0, 12) train_loss : 46.9074821472168 
+DLL 2023-07-12 19:51:53.337436 - (0, 12) train_items_per_sec : 18594.623770122114 items/s
+DLL 2023-07-12 19:51:53.337483 - (0, 12) train_iter_time : 1.464670667000064 
+DLL 2023-07-12 19:51:53.348916 - (0,) train_items_per_sec : 18395.683248143436 items/s
+DLL 2023-07-12 19:51:53.348971 - (0,) train_loss : 46.9074821472168 
+DLL 2023-07-12 19:51:53.348984 - (0,) train_epoch_time : 20.627870415000416 
+DLL 2023-07-12 19:51:54.456337 - (0, 13, 0) val_items_per_sec : 91673.0710297154 items/s
+DLL 2023-07-12 19:51:55.108027 - (0, 13, 1) val_items_per_sec : 90468.70568334985 items/s
+DLL 2023-07-12 19:51:55.273479 - (0, 13, 2) val_items_per_sec : 18423.14620168384 items/s
+DLL 2023-07-12 19:51:55.283287 - (0,) val_loss : 52.9111073811849 None
+DLL 2023-07-12 19:51:55.283326 - (0,) val_items_per_sec : 66854.97430491637 items/s
+Saving model and optimizer state at epoch 0 to ./checkpoint_Tacotron2_0.pt
+DLL 2023-07-12 19:51:56.655369 - (1, 0) glob_iter/iters_per_epoch : 13/13 
+DLL 2023-07-12 19:51:57.024664 - (1, 0) train_loss : 47.2700080871582 
+DLL 2023-07-12 19:51:58.093405 - (1, 0) train_items_per_sec : 17909.909101831054 items/s
+DLL 2023-07-12 19:51:58.093451 - (1, 0) train_iter_time : 1.4380865839998478 
+DLL 2023-07-12 19:51:58.100343 - (1, 1) glob_iter/iters_per_epoch : 14/13 
+DLL 2023-07-12 19:51:58.391384 - (1, 1) train_loss : 48.46968078613281 
+DLL 2023-07-12 19:51:59.491766 - (1, 1) train_items_per_sec : 19229.897740319262 items/s
+DLL 2023-07-12 19:51:59.491816 - (1, 1) train_iter_time : 1.3914270560003388 
+DLL 2023-07-12 19:51:59.495760 - (1, 2) glob_iter/iters_per_epoch : 15/13 
+DLL 2023-07-12 19:51:59.785476 - (1, 2) train_loss : 48.471588134765625 
+DLL 2023-07-12 19:52:00.890618 - (1, 2) train_items_per_sec : 19475.03971909992 items/s
+DLL 2023-07-12 19:52:00.890666 - (1, 2) train_iter_time : 1.3948623669998597 
+DLL 2023-07-12 19:52:00.891501 - (1, 3) glob_iter/iters_per_epoch : 16/13 
+DLL 2023-07-12 19:52:01.195076 - (1, 3) train_loss : 44.23277282714844 
+DLL 2023-07-12 19:52:02.351717 - (1, 3) train_items_per_sec : 17669.259915843624 items/s
+DLL 2023-07-12 19:52:02.351766 - (1, 3) train_iter_time : 1.4602196200003164 
+DLL 2023-07-12 19:52:02.356775 - (1, 4) glob_iter/iters_per_epoch : 17/13 
+DLL 2023-07-12 19:52:02.680178 - (1, 4) train_loss : 51.103397369384766 
+DLL 2023-07-12 19:52:03.800833 - (1, 4) train_items_per_sec : 20149.400495434093 items/s
+DLL 2023-07-12 19:52:03.800885 - (1, 4) train_iter_time : 1.444062815000052 
+DLL 2023-07-12 19:52:03.801707 - (1, 5) glob_iter/iters_per_epoch : 18/13 
+DLL 2023-07-12 19:52:04.124368 - (1, 5) train_loss : 44.03668212890625 
+DLL 2023-07-12 19:52:05.227022 - (1, 5) train_items_per_sec : 17450.129320177544 items/s
+DLL 2023-07-12 19:52:05.227071 - (1, 5) train_iter_time : 1.425318950000019 
+DLL 2023-07-12 19:52:05.227894 - (1, 6) glob_iter/iters_per_epoch : 19/13 
+DLL 2023-07-12 19:52:05.533242 - (1, 6) train_loss : 47.94136428833008 
+DLL 2023-07-12 19:52:06.687396 - (1, 6) train_items_per_sec : 19029.010380997657 items/s
+DLL 2023-07-12 19:52:06.687449 - (1, 6) train_iter_time : 1.4595083740000518 
+DLL 2023-07-12 19:52:06.692960 - (1, 7) glob_iter/iters_per_epoch : 20/13 
+DLL 2023-07-12 19:52:06.984218 - (1, 7) train_loss : 50.99168014526367 
+DLL 2023-07-12 19:52:08.082790 - (1, 7) train_items_per_sec : 20304.564451911807 items/s
+DLL 2023-07-12 19:52:08.082839 - (1, 7) train_iter_time : 1.389835279000181 
+DLL 2023-07-12 19:52:08.087027 - (1, 8) glob_iter/iters_per_epoch : 21/13 
+DLL 2023-07-12 19:52:08.388793 - (1, 8) train_loss : 44.60459518432617 
+DLL 2023-07-12 19:52:09.549036 - (1, 8) train_items_per_sec : 17555.954020380497 items/s
+DLL 2023-07-12 19:52:09.549088 - (1, 8) train_iter_time : 1.4620111199997154 
+DLL 2023-07-12 19:52:09.553950 - (1, 9) glob_iter/iters_per_epoch : 22/13 
+DLL 2023-07-12 19:52:09.858573 - (1, 9) train_loss : 48.70530700683594 
+DLL 2023-07-12 19:52:11.013974 - (1, 9) train_items_per_sec : 18987.29472522744 items/s
+DLL 2023-07-12 19:52:11.014024 - (1, 9) train_iter_time : 1.460028950999913 
+DLL 2023-07-12 19:52:11.017502 - (1, 10) glob_iter/iters_per_epoch : 23/13 
+DLL 2023-07-12 19:52:11.316645 - (1, 10) train_loss : 50.44378662109375 
+DLL 2023-07-12 19:52:12.473326 - (1, 10) train_items_per_sec : 20028.46838170536 items/s
+DLL 2023-07-12 19:52:12.473374 - (1, 10) train_iter_time : 1.4558277470000576 
+DLL 2023-07-12 19:52:12.474619 - (1, 11) glob_iter/iters_per_epoch : 24/13 
+DLL 2023-07-12 19:52:12.776182 - (1, 11) train_loss : 45.49681854248047 
+DLL 2023-07-12 19:52:13.932638 - (1, 11) train_items_per_sec : 18217.13593544406 items/s
+DLL 2023-07-12 19:52:13.932687 - (1, 11) train_iter_time : 1.4580228249997162 
+DLL 2023-07-12 19:52:13.937320 - (1, 12) glob_iter/iters_per_epoch : 25/13 
+DLL 2023-07-12 19:52:14.268329 - (1, 12) train_loss : 47.370609283447266 
+DLL 2023-07-12 19:52:15.428125 - (1, 12) train_items_per_sec : 18426.237561169855 items/s
+DLL 2023-07-12 19:52:15.428175 - (1, 12) train_iter_time : 1.4908089569998992 
+DLL 2023-07-12 19:52:15.447397 - (1,) train_items_per_sec : 18802.484749964784 items/s
+DLL 2023-07-12 19:52:15.447448 - (1,) train_loss : 47.370609283447266 
+DLL 2023-07-12 19:52:15.447460 - (1,) train_epoch_time : 19.95938218099991 
+DLL 2023-07-12 19:52:16.586765 - (1, 26, 0) val_items_per_sec : 89056.07180922596 items/s
+DLL 2023-07-12 19:52:17.235502 - (1, 26, 1) val_items_per_sec : 92380.70906544523 items/s
+DLL 2023-07-12 19:52:17.401986 - (1, 26, 2) val_items_per_sec : 18292.214312009335 items/s
+DLL 2023-07-12 19:52:17.411987 - (1,) val_loss : 52.90406290690104 None
+DLL 2023-07-12 19:52:17.412027 - (1,) val_items_per_sec : 66576.3317288935 items/s
+DLL 2023-07-12 19:52:17.412401 - () run_time : 45.28354268699968 s
+DLL 2023-07-12 19:52:17.412421 - () val_loss : 52.90406290690104 None
+DLL 2023-07-12 19:52:17.412431 - () train_loss : 47.370609283447266 
+DLL 2023-07-12 19:52:17.412439 - () train_items_per_sec : 18802.484749964784 items/s
+DLL 2023-07-12 19:52:17.412448 - () val_items_per_sec : 66576.3317288935 items/s
+DONE!
diff --git a/pytorch/results/4090laptop_v1/PyTorch_tacotron2_FP16/benchmark.para b/pytorch/results/4090laptop_v1/PyTorch_tacotron2_FP16/benchmark.para
new file mode 100755
index 00000000..e35db812
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_tacotron2_FP16/benchmark.para
@@ -0,0 +1,2 @@
+GLOBAL_BATCH 48
+GPU 1
diff --git a/pytorch/results/4090laptop_v1/PyTorch_tacotron2_FP32/12-07-2023_20-12-14.txt b/pytorch/results/4090laptop_v1/PyTorch_tacotron2_FP32/12-07-2023_20-12-14.txt
new file mode 100755
index 00000000..b5b976ef
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_tacotron2_FP32/12-07-2023_20-12-14.txt
@@ -0,0 +1,170 @@
+OMP: Info #276: omp_set_nested routine deprecated, please use omp_set_max_active_levels instead.
+DLL 2023-07-12 20:12:22.008105 - PARAMETER output : ./ 
+DLL 2023-07-12 20:12:22.008139 - PARAMETER dataset_path : /data/tacotron2/LJSpeech-1.1 
+DLL 2023-07-12 20:12:22.008150 - PARAMETER model_name : Tacotron2 
+DLL 2023-07-12 20:12:22.008158 - PARAMETER log_file : nvlog.json 
+DLL 2023-07-12 20:12:22.008165 - PARAMETER anneal_steps : None 
+DLL 2023-07-12 20:12:22.008172 - PARAMETER anneal_factor : 0.1 
+DLL 2023-07-12 20:12:22.008179 - PARAMETER config_file : None 
+DLL 2023-07-12 20:12:22.008186 - PARAMETER seed : None 
+DLL 2023-07-12 20:12:22.008192 - PARAMETER epochs : 2 
+DLL 2023-07-12 20:12:22.008199 - PARAMETER epochs_per_checkpoint : 50 
+DLL 2023-07-12 20:12:22.008205 - PARAMETER checkpoint_path :  
+DLL 2023-07-12 20:12:22.008212 - PARAMETER resume_from_last : False 
+DLL 2023-07-12 20:12:22.008219 - PARAMETER dynamic_loss_scaling : True 
+DLL 2023-07-12 20:12:22.008225 - PARAMETER amp : False 
+DLL 2023-07-12 20:12:22.008231 - PARAMETER cudnn_enabled : True 
+DLL 2023-07-12 20:12:22.008237 - PARAMETER cudnn_benchmark : False 
+DLL 2023-07-12 20:12:22.008243 - PARAMETER disable_uniform_initialize_bn_weight : False 
+DLL 2023-07-12 20:12:22.008249 - PARAMETER use_saved_learning_rate : False 
+DLL 2023-07-12 20:12:22.008255 - PARAMETER learning_rate : 0.0 
+DLL 2023-07-12 20:12:22.008262 - PARAMETER weight_decay : 1e-06 
+DLL 2023-07-12 20:12:22.008269 - PARAMETER grad_clip_thresh : 1.0 
+DLL 2023-07-12 20:12:22.008276 - PARAMETER batch_size : 48 
+DLL 2023-07-12 20:12:22.008282 - PARAMETER grad_clip : 5.0 
+DLL 2023-07-12 20:12:22.008288 - PARAMETER load_mel_from_disk : False 
+DLL 2023-07-12 20:12:22.008294 - PARAMETER training_files : filelists/ljs_audio_text_train_subset_625_filelist.txt 
+DLL 2023-07-12 20:12:22.008301 - PARAMETER validation_files : filelists/ljs_audio_text_val_filelist.txt 
+DLL 2023-07-12 20:12:22.008307 - PARAMETER text_cleaners : ['english_cleaners'] 
+DLL 2023-07-12 20:12:22.008315 - PARAMETER max_wav_value : 32768.0 
+DLL 2023-07-12 20:12:22.008322 - PARAMETER sampling_rate : 22050 
+DLL 2023-07-12 20:12:22.008328 - PARAMETER filter_length : 1024 
+DLL 2023-07-12 20:12:22.008333 - PARAMETER hop_length : 256 
+DLL 2023-07-12 20:12:22.008339 - PARAMETER win_length : 1024 
+DLL 2023-07-12 20:12:22.008345 - PARAMETER mel_fmin : 0.0 
+DLL 2023-07-12 20:12:22.008351 - PARAMETER mel_fmax : 8000.0 
+DLL 2023-07-12 20:12:22.008357 - PARAMETER rank : 0 
+DLL 2023-07-12 20:12:22.008363 - PARAMETER world_size : 1 
+DLL 2023-07-12 20:12:22.008369 - PARAMETER dist_url : tcp://localhost:23456 
+DLL 2023-07-12 20:12:22.008375 - PARAMETER group_name : group_name 
+DLL 2023-07-12 20:12:22.008381 - PARAMETER dist_backend : nccl 
+DLL 2023-07-12 20:12:22.008387 - PARAMETER bench_class :  
+DLL 2023-07-12 20:12:22.008393 - PARAMETER model_name : Tacotron2_PyT 
+DLL 2023-07-12 20:12:23.995210 - (0, 0) glob_iter/iters_per_epoch : 0/13 
+DLL 2023-07-12 20:12:24.779796 - (0, 0) train_loss : 46.995784759521484 
+DLL 2023-07-12 20:12:25.971451 - (0, 0) train_items_per_sec : 13700.414433135593 items/s
+DLL 2023-07-12 20:12:25.971494 - (0, 0) train_iter_time : 1.9762905809998301 
+DLL 2023-07-12 20:12:25.973917 - (0, 1) glob_iter/iters_per_epoch : 1/13 
+DLL 2023-07-12 20:12:26.275910 - (0, 1) train_loss : 47.23094177246094 
+DLL 2023-07-12 20:12:27.430691 - (0, 1) train_items_per_sec : 18557.380946484263 items/s
+DLL 2023-07-12 20:12:27.430735 - (0, 1) train_iter_time : 1.4567788459999065 
+DLL 2023-07-12 20:12:27.431611 - (0, 2) glob_iter/iters_per_epoch : 2/13 
+DLL 2023-07-12 20:12:27.721178 - (0, 2) train_loss : 46.15873336791992 
+DLL 2023-07-12 20:12:28.812737 - (0, 2) train_items_per_sec : 18596.385189702207 items/s
+DLL 2023-07-12 20:12:28.812782 - (0, 2) train_iter_time : 1.3811286299996937 
+DLL 2023-07-12 20:12:28.813651 - (0, 3) glob_iter/iters_per_epoch : 3/13 
+DLL 2023-07-12 20:12:29.109874 - (0, 3) train_loss : 47.86547088623047 
+DLL 2023-07-12 20:12:30.235634 - (0, 3) train_items_per_sec : 18834.9300729743 items/s
+DLL 2023-07-12 20:12:30.235678 - (0, 3) train_iter_time : 1.4219856350000555 
+DLL 2023-07-12 20:12:30.239489 - (0, 4) glob_iter/iters_per_epoch : 4/13 
+DLL 2023-07-12 20:12:30.549144 - (0, 4) train_loss : 47.152042388916016 
+DLL 2023-07-12 20:12:31.723212 - (0, 4) train_items_per_sec : 18132.036103901228 items/s
+DLL 2023-07-12 20:12:31.723257 - (0, 4) train_iter_time : 1.4837274670003353 
+DLL 2023-07-12 20:12:31.724151 - (0, 5) glob_iter/iters_per_epoch : 5/13 
+DLL 2023-07-12 20:12:32.001470 - (0, 5) train_loss : 46.30691146850586 
+DLL 2023-07-12 20:12:33.083685 - (0, 5) train_items_per_sec : 18469.509617400126 items/s
+DLL 2023-07-12 20:12:33.083733 - (0, 5) train_iter_time : 1.359537991000252 
+DLL 2023-07-12 20:12:33.087859 - (0, 6) glob_iter/iters_per_epoch : 6/13 
+DLL 2023-07-12 20:12:33.422718 - (0, 6) train_loss : 52.295326232910156 
+DLL 2023-07-12 20:12:34.533304 - (0, 6) train_items_per_sec : 20423.392273447305 items/s
+DLL 2023-07-12 20:12:34.533349 - (0, 6) train_iter_time : 1.4454503739998472 
+DLL 2023-07-12 20:12:34.534224 - (0, 7) glob_iter/iters_per_epoch : 7/13 
+DLL 2023-07-12 20:12:34.826256 - (0, 7) train_loss : 46.274051666259766 
+DLL 2023-07-12 20:12:35.920294 - (0, 7) train_items_per_sec : 18752.26201595308 items/s
+DLL 2023-07-12 20:12:35.920343 - (0, 7) train_iter_time : 1.3860727830001451 
+DLL 2023-07-12 20:12:35.924609 - (0, 8) glob_iter/iters_per_epoch : 8/13 
+DLL 2023-07-12 20:12:36.221425 - (0, 8) train_loss : 47.64881896972656 
+DLL 2023-07-12 20:12:37.358977 - (0, 8) train_items_per_sec : 19046.64339682188 items/s
+DLL 2023-07-12 20:12:37.359023 - (0, 8) train_iter_time : 1.4343734710000717 
+DLL 2023-07-12 20:12:37.364344 - (0, 9) glob_iter/iters_per_epoch : 9/13 
+DLL 2023-07-12 20:12:37.670737 - (0, 9) train_loss : 47.43180465698242 
+DLL 2023-07-12 20:12:38.838762 - (0, 9) train_items_per_sec : 18743.60866750463 items/s
+DLL 2023-07-12 20:12:38.838806 - (0, 9) train_iter_time : 1.4744225879999249 
+DLL 2023-07-12 20:12:38.839700 - (0, 10) glob_iter/iters_per_epoch : 10/13 
+DLL 2023-07-12 20:12:39.152015 - (0, 10) train_loss : 48.39072799682617 
+DLL 2023-07-12 20:12:40.307752 - (0, 10) train_items_per_sec : 19179.78576978405 items/s
+DLL 2023-07-12 20:12:40.307799 - (0, 10) train_iter_time : 1.468056021999928 
+DLL 2023-07-12 20:12:40.312267 - (0, 11) glob_iter/iters_per_epoch : 11/13 
+DLL 2023-07-12 20:12:40.622065 - (0, 11) train_loss : 47.73684310913086 
+DLL 2023-07-12 20:12:41.800903 - (0, 11) train_items_per_sec : 18625.695428351224 items/s
+DLL 2023-07-12 20:12:41.800950 - (0, 11) train_iter_time : 1.4886424029996306 
+DLL 2023-07-12 20:12:41.804614 - (0, 12) glob_iter/iters_per_epoch : 12/13 
+DLL 2023-07-12 20:12:42.109740 - (0, 12) train_loss : 48.258758544921875 
+DLL 2023-07-12 20:12:43.266739 - (0, 12) train_items_per_sec : 18699.40658434621 items/s
+DLL 2023-07-12 20:12:43.266783 - (0, 12) train_iter_time : 1.462131959999624 
+DLL 2023-07-12 20:12:43.282554 - (0,) train_items_per_sec : 18443.188499985084 items/s
+DLL 2023-07-12 20:12:43.282616 - (0,) train_loss : 48.258758544921875 
+DLL 2023-07-12 20:12:43.282648 - (0,) train_epoch_time : 20.53273155199986 
+DLL 2023-07-12 20:12:44.396424 - (0, 13, 0) val_items_per_sec : 92212.90840914207 items/s
+DLL 2023-07-12 20:12:45.060677 - (0, 13, 1) val_items_per_sec : 89875.39201232395 items/s
+DLL 2023-07-12 20:12:45.229078 - (0, 13, 2) val_items_per_sec : 18217.120284989665 items/s
+DLL 2023-07-12 20:12:45.243016 - (0,) val_loss : 52.86393483479818 None
+DLL 2023-07-12 20:12:45.243069 - (0,) val_items_per_sec : 66768.47356881855 items/s
+Saving model and optimizer state at epoch 0 to ./checkpoint_Tacotron2_0.pt
+Updating symlink ./checkpoint_Tacotron2_last.pt to point to checkpoint_Tacotron2_0.pt
+DLL 2023-07-12 20:12:46.801246 - (1, 0) glob_iter/iters_per_epoch : 13/13 
+DLL 2023-07-12 20:12:47.241545 - (1, 0) train_loss : 46.01240539550781 
+DLL 2023-07-12 20:12:48.419683 - (1, 0) train_items_per_sec : 16176.105142378878 items/s
+DLL 2023-07-12 20:12:48.419735 - (1, 0) train_iter_time : 1.6184983819998706 
+DLL 2023-07-12 20:12:48.425250 - (1, 1) glob_iter/iters_per_epoch : 14/13 
+DLL 2023-07-12 20:12:48.731795 - (1, 1) train_loss : 47.13193130493164 
+DLL 2023-07-12 20:12:49.902226 - (1, 1) train_items_per_sec : 18570.321107257874 items/s
+DLL 2023-07-12 20:12:49.902272 - (1, 1) train_iter_time : 1.476980383999944 
+DLL 2023-07-12 20:12:49.903160 - (1, 2) glob_iter/iters_per_epoch : 15/13 
+DLL 2023-07-12 20:12:50.199114 - (1, 2) train_loss : 46.19404602050781 
+DLL 2023-07-12 20:12:51.331325 - (1, 2) train_items_per_sec : 18319.295027817843 items/s
+DLL 2023-07-12 20:12:51.331374 - (1, 2) train_iter_time : 1.4281663110000409 
+DLL 2023-07-12 20:12:51.337931 - (1, 3) glob_iter/iters_per_epoch : 16/13 
+DLL 2023-07-12 20:12:51.638783 - (1, 3) train_loss : 47.60956954956055 
+DLL 2023-07-12 20:12:52.796811 - (1, 3) train_items_per_sec : 18941.83642757372 items/s
+DLL 2023-07-12 20:12:52.796867 - (1, 3) train_iter_time : 1.4588870570000836 
+DLL 2023-07-12 20:12:52.797742 - (1, 4) glob_iter/iters_per_epoch : 17/13 
+DLL 2023-07-12 20:12:53.092763 - (1, 4) train_loss : 49.60683822631836 
+DLL 2023-07-12 20:12:54.209373 - (1, 4) train_items_per_sec : 19718.98992802068 items/s
+DLL 2023-07-12 20:12:54.209428 - (1, 4) train_iter_time : 1.4116341710000597 
+DLL 2023-07-12 20:12:54.213907 - (1, 5) glob_iter/iters_per_epoch : 18/13 
+DLL 2023-07-12 20:12:54.521194 - (1, 5) train_loss : 48.00442886352539 
+DLL 2023-07-12 20:12:55.696893 - (1, 5) train_items_per_sec : 18821.77710372485 items/s
+DLL 2023-07-12 20:12:55.696939 - (1, 5) train_iter_time : 1.483016181000039 
+DLL 2023-07-12 20:12:55.700726 - (1, 6) glob_iter/iters_per_epoch : 19/13 
+DLL 2023-07-12 20:12:56.005480 - (1, 6) train_loss : 50.07993698120117 
+DLL 2023-07-12 20:12:57.179666 - (1, 6) train_items_per_sec : 19490.942862247713 items/s
+DLL 2023-07-12 20:12:57.179718 - (1, 6) train_iter_time : 1.4789433329997337 
+DLL 2023-07-12 20:12:57.180602 - (1, 7) glob_iter/iters_per_epoch : 20/13 
+DLL 2023-07-12 20:12:57.463515 - (1, 7) train_loss : 49.34697723388672 
+DLL 2023-07-12 20:12:58.525653 - (1, 7) train_items_per_sec : 19813.321442876673 items/s
+DLL 2023-07-12 20:12:58.525700 - (1, 7) train_iter_time : 1.345054643000367 
+DLL 2023-07-12 20:12:58.530223 - (1, 8) glob_iter/iters_per_epoch : 21/13 
+DLL 2023-07-12 20:12:58.826697 - (1, 8) train_loss : 44.6431884765625 
+DLL 2023-07-12 20:12:59.963423 - (1, 8) train_items_per_sec : 17843.229992503584 items/s
+DLL 2023-07-12 20:12:59.963471 - (1, 8) train_iter_time : 1.433204638999996 
+DLL 2023-07-12 20:12:59.966841 - (1, 9) glob_iter/iters_per_epoch : 22/13 
+DLL 2023-07-12 20:13:00.295683 - (1, 9) train_loss : 47.72003936767578 
+DLL 2023-07-12 20:13:01.429830 - (1, 9) train_items_per_sec : 18538.659343295418 items/s
+DLL 2023-07-12 20:13:01.429875 - (1, 9) train_iter_time : 1.462996837999981 
+DLL 2023-07-12 20:13:01.430723 - (1, 10) glob_iter/iters_per_epoch : 23/13 
+DLL 2023-07-12 20:13:01.731350 - (1, 10) train_loss : 43.848323822021484 
+DLL 2023-07-12 20:13:02.847208 - (1, 10) train_items_per_sec : 17453.0241339953 items/s
+DLL 2023-07-12 20:13:02.847254 - (1, 10) train_iter_time : 1.4164880430002995 
+DLL 2023-07-12 20:13:02.852445 - (1, 11) glob_iter/iters_per_epoch : 24/13 
+DLL 2023-07-12 20:13:03.153991 - (1, 11) train_loss : 48.00401306152344 
+DLL 2023-07-12 20:13:04.299282 - (1, 11) train_items_per_sec : 19203.191173372605 items/s
+DLL 2023-07-12 20:13:04.299332 - (1, 11) train_iter_time : 1.4468428579998545 
+DLL 2023-07-12 20:13:04.302995 - (1, 12) glob_iter/iters_per_epoch : 25/13 
+DLL 2023-07-12 20:13:04.606200 - (1, 12) train_loss : 48.876953125 
+DLL 2023-07-12 20:13:05.776471 - (1, 12) train_items_per_sec : 19166.8513658557 items/s
+DLL 2023-07-12 20:13:05.776517 - (1, 12) train_iter_time : 1.473481453000204 
+DLL 2023-07-12 20:13:05.798447 - (1,) train_items_per_sec : 18619.811157763143 items/s
+DLL 2023-07-12 20:13:05.798493 - (1,) train_loss : 48.876953125 
+DLL 2023-07-12 20:13:05.798505 - (1,) train_epoch_time : 20.20957270100007 
+DLL 2023-07-12 20:13:06.946394 - (1, 26, 0) val_items_per_sec : 93083.85745463734 items/s
+DLL 2023-07-12 20:13:07.605546 - (1, 26, 1) val_items_per_sec : 92577.17352364655 items/s
+DLL 2023-07-12 20:13:07.772641 - (1, 26, 2) val_items_per_sec : 18291.570700806224 items/s
+DLL 2023-07-12 20:13:07.786394 - (1,) val_loss : 52.816715240478516 None
+DLL 2023-07-12 20:13:07.786436 - (1,) val_items_per_sec : 67984.2005596967 items/s
+DLL 2023-07-12 20:13:07.786848 - () run_time : 45.66419510700007 s
+DLL 2023-07-12 20:13:07.786866 - () val_loss : 52.816715240478516 None
+DLL 2023-07-12 20:13:07.786877 - () train_loss : 48.876953125 
+DLL 2023-07-12 20:13:07.786886 - () train_items_per_sec : 18619.811157763143 items/s
+DLL 2023-07-12 20:13:07.786893 - () val_items_per_sec : 67984.2005596967 items/s
+DONE!
diff --git a/pytorch/results/4090laptop_v1/PyTorch_tacotron2_FP32/benchmark.para b/pytorch/results/4090laptop_v1/PyTorch_tacotron2_FP32/benchmark.para
new file mode 100755
index 00000000..e35db812
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_tacotron2_FP32/benchmark.para
@@ -0,0 +1,2 @@
+GLOBAL_BATCH 48
+GPU 1
diff --git a/pytorch/results/4090laptop_v1/PyTorch_transformerxlbase_FP16/12-07-2023_20-08-21.txt b/pytorch/results/4090laptop_v1/PyTorch_transformerxlbase_FP16/12-07-2023_20-08-21.txt
new file mode 100755
index 00000000..b5977462
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_transformerxlbase_FP16/12-07-2023_20-08-21.txt
@@ -0,0 +1,202 @@
+0: thread affinity: {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}
+Experiment dir : LM-TFM
+Namespace(adaptive=False, affinity='socket_unique_interleaved', amp='apex', apex_amp_opt_level='O2', append_dataset=False, append_time=False, attn_type=0, batch_chunk=1, batch_size=12, clamp_len=-1, clip=0.25, clip_nonemb=False, cuda=True, d_embed=512, d_head=64, d_inner=2048, d_model=512, data='/data/transformer-xl/wikitext-103', dataset='wt103', debug=False, decay_rate=0.5, div_val=1, dllog_file='train_log.json', dropatt=0.0, dropout=0.1, emb_init='normal', emb_init_range=0.01, eta_min=0.001, eval_batch_size=16, eval_interval=5000, eval_max_steps=-1, eval_tgt_len=192, ext_len=0, fp16=True, gpu0_bsz=-1, init='normal', init_range=0.1, init_std=0.02, local_batch_size=None, local_rank=0, log_all_ranks=False, log_interval=10, lr=0.0, lr_min=0.0, max_step=400, max_step_scheduler=None, mem_len=192, mom=0.0, multi_gpu=None, n_head=8, n_layer=16, no_env=False, no_eval=True, no_test=False, not_tied=False, optim='jitlamb', patience=0, pre_lnorm=False, proj_init_std=0.01, restart='', roll=True, same_length=False, sample_softmax=-1, save_all=False, scheduler='cosine', seed=1111, swap_mem=False, target_perplexity=None, target_throughput=None, tgt_len=192, tied=True, txtlog_file='train_log.log', varlen=False, vocab='word', warmup_step=1000, weight_decay=0.0, work_dir='LM-TFM')
+world size: 1
+Collecting environment information...
+PyTorch version: 1.13.0a0+d0d6b1f
+Is debug build: False
+CUDA used to build PyTorch: 11.8
+ROCM used to build PyTorch: N/A
+
+OS: Ubuntu 20.04.5 LTS (x86_64)
+GCC version: (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0
+Clang version: Could not collect
+CMake version: version 3.22.2
+Libc version: glibc-2.31
+
+Python version: 3.8.13 | packaged by conda-forge | (default, Mar 25 2022, 06:04:10)  [GCC 10.3.0] (64-bit runtime)
+Python platform: Linux-5.19.0-46-generic-x86_64-with-glibc2.10
+Is CUDA available: True
+CUDA runtime version: 11.8.89
+GPU models and configuration: GPU 0: NVIDIA GeForce RTX 4090 Laptop GPU
+Nvidia driver version: 525.116.04
+cuDNN version: Probably one of the following:
+/usr/lib/x86_64-linux-gnu/libcudnn.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.6.0
+HIP runtime version: N/A
+MIOpen runtime version: N/A
+Is XNNPACK available: True
+
+Versions of relevant libraries:
+[pip3] functorch==0.3.0a0
+[pip3] numpy==1.22.2
+[pip3] pytorch-quantization==2.1.2
+[pip3] pytorch-transformers==1.1.0
+[pip3] torch==1.13.0a0+d0d6b1f
+[pip3] torch-tensorrt==1.3.0a0
+[pip3] torchtext==0.11.0a0
+[pip3] torchvision==0.14.0a0
+[conda] functorch                 0.3.0a0                  pypi_0    pypi
+[conda] mkl                       2020.4             h726a3e6_304    conda-forge
+[conda] mkl-include               2020.4             h726a3e6_304    conda-forge
+[conda] numpy                     1.22.2           py38h6ae9a64_0    conda-forge
+[conda] pytorch-quantization      2.1.2                    pypi_0    pypi
+[conda] pytorch-transformers      1.1.0                    pypi_0    pypi
+[conda] torch                     1.13.0a0+d0d6b1f          pypi_0    pypi
+[conda] torch-tensorrt            1.3.0a0                  pypi_0    pypi
+[conda] torchtext                 0.11.0a0                 pypi_0    pypi
+[conda] torchvision               0.14.0a0                 pypi_0    pypi
+Loading cached dataset...
+Selected optimization level O2:  FP16 training with FP32 batchnorm and FP32 master weights.
+
+Defaults for this optimization level are:
+enabled                : True
+opt_level              : O2
+cast_model_type        : torch.float16
+patch_torch_functions  : False
+keep_batchnorm_fp32    : True
+master_weights         : True
+loss_scale             : dynamic
+Processing user overrides (additional kwargs that are not None)...
+After processing overrides, optimization options are:
+enabled                : True
+opt_level              : O2
+cast_model_type        : torch.float16
+patch_torch_functions  : False
+keep_batchnorm_fp32    : True
+master_weights         : True
+loss_scale             : dynamic
+====================================================================================================
+    - work_dir : LM-TFM
+    - append_dataset : False
+    - append_time : False
+    - cuda : True
+    - fp16 : True
+    - restart : 
+    - debug : False
+    - log_all_ranks : False
+    - dllog_file : train_log.json
+    - txtlog_file : train_log.log
+    - save_all : False
+    - no_env : False
+    - no_eval : True
+    - no_test : False
+    - log_interval : 10
+    - target_throughput : None
+    - target_perplexity : None
+    - apex_amp_opt_level : O2
+    - amp : apex
+    - affinity : socket_unique_interleaved
+    - data : /data/transformer-xl/wikitext-103
+    - dataset : wt103
+    - vocab : word
+    - n_layer : 16
+    - n_head : 8
+    - d_head : 64
+    - d_embed : 512
+    - d_model : 512
+    - d_inner : 2048
+    - dropout : 0.1
+    - dropatt : 0.0
+    - pre_lnorm : False
+    - attn_type : 0
+    - not_tied : False
+    - clamp_len : -1
+    - adaptive : False
+    - div_val : 1
+    - sample_softmax : -1
+    - init : normal
+    - emb_init : normal
+    - init_range : 0.1
+    - emb_init_range : 0.01
+    - init_std : 0.02
+    - proj_init_std : 0.01
+    - optim : jitlamb
+    - lr : 0.0
+    - mom : 0.0
+    - scheduler : cosine
+    - max_step_scheduler : None
+    - warmup_step : 1000
+    - decay_rate : 0.5
+    - lr_min : 0.0
+    - clip : 0.25
+    - weight_decay : 0.0
+    - clip_nonemb : False
+    - patience : 0
+    - eta_min : 0.001
+    - max_step : 400
+    - batch_size : 12
+    - local_batch_size : None
+    - batch_chunk : 1
+    - roll : True
+    - tgt_len : 192
+    - ext_len : 0
+    - mem_len : 192
+    - seed : 1111
+    - multi_gpu : None
+    - gpu0_bsz : -1
+    - same_length : False
+    - varlen : False
+    - swap_mem : False
+    - eval_tgt_len : 192
+    - eval_batch_size : 16
+    - eval_max_steps : -1
+    - eval_interval : 5000
+    - local_rank : 0
+    - tied : True
+    - n_token : 267735
+    - n_all_param : 191948759
+    - n_nonemb_param : 54599680
+====================================================================================================
+#params = 191948759
+#non emb params = 54599680
+| epoch   1 step       10 | batches     10 / 44804 | lr 0.000e+00 | ms/batch 184.0 | tok/s   12522 | loss 12.55 | ppl 282907.95
+| epoch   1 step       20 | batches     20 / 44804 | lr 0.000e+00 | ms/batch 129.9 | tok/s   17742 | loss 12.56 | ppl 284347.54
+| epoch   1 step       30 | batches     30 / 44804 | lr 0.000e+00 | ms/batch 130.2 | tok/s   17692 | loss 12.56 | ppl 285007.20
+| epoch   1 step       40 | batches     40 / 44804 | lr 0.000e+00 | ms/batch 129.9 | tok/s   17734 | loss 12.55 | ppl 282304.05
+| epoch   1 step       50 | batches     50 / 44804 | lr 0.000e+00 | ms/batch 130.2 | tok/s   17695 | loss 12.56 | ppl 284153.85
+| epoch   1 step       60 | batches     60 / 44804 | lr 0.000e+00 | ms/batch 130.8 | tok/s   17614 | loss 12.56 | ppl 283811.80
+| epoch   1 step       70 | batches     70 / 44804 | lr 0.000e+00 | ms/batch 130.6 | tok/s   17646 | loss 12.55 | ppl 282319.53
+| epoch   1 step       80 | batches     80 / 44804 | lr 0.000e+00 | ms/batch 130.8 | tok/s   17618 | loss 12.57 | ppl 286871.11
+| epoch   1 step       90 | batches     90 / 44804 | lr 0.000e+00 | ms/batch 130.9 | tok/s   17597 | loss 12.57 | ppl 287155.94
+| epoch   1 step      100 | batches    100 / 44804 | lr 0.000e+00 | ms/batch 130.5 | tok/s   17650 | loss 12.57 | ppl 286367.20
+| epoch   1 step      110 | batches    110 / 44804 | lr 0.000e+00 | ms/batch 131.1 | tok/s   17572 | loss 12.55 | ppl 280912.66
+| epoch   1 step      120 | batches    120 / 44804 | lr 0.000e+00 | ms/batch 131.2 | tok/s   17562 | loss 12.56 | ppl 284113.31
+| epoch   1 step      130 | batches    130 / 44804 | lr 0.000e+00 | ms/batch 130.8 | tok/s   17620 | loss 12.55 | ppl 282614.10
+| epoch   1 step      140 | batches    140 / 44804 | lr 0.000e+00 | ms/batch 131.6 | tok/s   17505 | loss 12.58 | ppl 289998.49
+| epoch   1 step      150 | batches    150 / 44804 | lr 0.000e+00 | ms/batch 131.2 | tok/s   17564 | loss 12.56 | ppl 285595.97
+| epoch   1 step      160 | batches    160 / 44804 | lr 0.000e+00 | ms/batch 130.9 | tok/s   17596 | loss 12.56 | ppl 284855.47
+| epoch   1 step      170 | batches    170 / 44804 | lr 0.000e+00 | ms/batch 131.3 | tok/s   17544 | loss 12.55 | ppl 281945.91
+| epoch   1 step      180 | batches    180 / 44804 | lr 0.000e+00 | ms/batch 131.4 | tok/s   17531 | loss 12.54 | ppl 279338.81
+| epoch   1 step      190 | batches    190 / 44804 | lr 0.000e+00 | ms/batch 131.3 | tok/s   17547 | loss 12.53 | ppl 277534.11
+| epoch   1 step      200 | batches    200 / 44804 | lr 0.000e+00 | ms/batch 131.3 | tok/s   17552 | loss 12.55 | ppl 281930.72
+| epoch   1 step      210 | batches    210 / 44804 | lr 0.000e+00 | ms/batch 131.8 | tok/s   17486 | loss 12.55 | ppl 281945.59
+| epoch   1 step      220 | batches    220 / 44804 | lr 0.000e+00 | ms/batch 131.4 | tok/s   17530 | loss 12.55 | ppl 280830.32
+| epoch   1 step      230 | batches    230 / 44804 | lr 0.000e+00 | ms/batch 131.4 | tok/s   17535 | loss 12.57 | ppl 287552.09
+| epoch   1 step      240 | batches    240 / 44804 | lr 0.000e+00 | ms/batch 131.4 | tok/s   17533 | loss 12.53 | ppl 277642.04
+| epoch   1 step      250 | batches    250 / 44804 | lr 0.000e+00 | ms/batch 131.8 | tok/s   17488 | loss 12.55 | ppl 283416.52
+| epoch   1 step      260 | batches    260 / 44804 | lr 0.000e+00 | ms/batch 131.6 | tok/s   17504 | loss 12.54 | ppl 279081.69
+| epoch   1 step      270 | batches    270 / 44804 | lr 0.000e+00 | ms/batch 131.1 | tok/s   17575 | loss 12.55 | ppl 282717.05
+| epoch   1 step      280 | batches    280 / 44804 | lr 0.000e+00 | ms/batch 131.9 | tok/s   17472 | loss 12.56 | ppl 284054.71
+| epoch   1 step      290 | batches    290 / 44804 | lr 0.000e+00 | ms/batch 131.8 | tok/s   17475 | loss 12.54 | ppl 277915.19
+| epoch   1 step      300 | batches    300 / 44804 | lr 0.000e+00 | ms/batch 131.8 | tok/s   17482 | loss 12.56 | ppl 284371.84
+| epoch   1 step      310 | batches    310 / 44804 | lr 0.000e+00 | ms/batch 131.7 | tok/s   17494 | loss 12.56 | ppl 283722.30
+| epoch   1 step      320 | batches    320 / 44804 | lr 0.000e+00 | ms/batch 131.9 | tok/s   17474 | loss 12.54 | ppl 279118.37
+| epoch   1 step      330 | batches    330 / 44804 | lr 0.000e+00 | ms/batch 132.3 | tok/s   17410 | loss 12.56 | ppl 283736.97
+| epoch   1 step      340 | batches    340 / 44804 | lr 0.000e+00 | ms/batch 131.8 | tok/s   17479 | loss 12.54 | ppl 278166.62
+| epoch   1 step      350 | batches    350 / 44804 | lr 0.000e+00 | ms/batch 132.0 | tok/s   17454 | loss 12.54 | ppl 279179.25
+| epoch   1 step      360 | batches    360 / 44804 | lr 0.000e+00 | ms/batch 132.0 | tok/s   17451 | loss 12.56 | ppl 283563.79
+| epoch   1 step      370 | batches    370 / 44804 | lr 0.000e+00 | ms/batch 132.0 | tok/s   17449 | loss 12.55 | ppl 283250.91
+| epoch   1 step      380 | batches    380 / 44804 | lr 0.000e+00 | ms/batch 132.3 | tok/s   17415 | loss 12.55 | ppl 282427.06
+| epoch   1 step      390 | batches    390 / 44804 | lr 0.000e+00 | ms/batch 132.0 | tok/s   17460 | loss 12.57 | ppl 286508.70
+| epoch   1 step      400 | batches    400 / 44804 | lr 0.000e+00 | ms/batch 132.1 | tok/s   17435 | loss 12.57 | ppl 286773.15
+----------------------------------------------------------------------------------------------------
+End of training
+Training time: 0.89 minutes
+Training throughput: 17533.76 tok/s
+DONE!
diff --git a/pytorch/results/4090laptop_v1/PyTorch_transformerxlbase_FP16/benchmark.para b/pytorch/results/4090laptop_v1/PyTorch_transformerxlbase_FP16/benchmark.para
new file mode 100755
index 00000000..8303d9e6
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_transformerxlbase_FP16/benchmark.para
@@ -0,0 +1,2 @@
+GLOBAL_BATCH 12
+GPU 1
diff --git a/pytorch/results/4090laptop_v1/PyTorch_transformerxlbase_FP32/12-07-2023_20-17-16.txt b/pytorch/results/4090laptop_v1/PyTorch_transformerxlbase_FP32/12-07-2023_20-17-16.txt
new file mode 100755
index 00000000..19045960
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_transformerxlbase_FP32/12-07-2023_20-17-16.txt
@@ -0,0 +1,183 @@
+0: thread affinity: {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}
+Experiment dir : LM-TFM
+Namespace(adaptive=False, affinity='socket_unique_interleaved', amp='apex', apex_amp_opt_level='O2', append_dataset=False, append_time=False, attn_type=0, batch_chunk=1, batch_size=8, clamp_len=-1, clip=0.25, clip_nonemb=False, cuda=True, d_embed=512, d_head=64, d_inner=2048, d_model=512, data='/data/transformer-xl/wikitext-103', dataset='wt103', debug=False, decay_rate=0.5, div_val=1, dllog_file='train_log.json', dropatt=0.0, dropout=0.1, emb_init='normal', emb_init_range=0.01, eta_min=0.001, eval_batch_size=16, eval_interval=5000, eval_max_steps=-1, eval_tgt_len=192, ext_len=0, fp16=False, gpu0_bsz=-1, init='normal', init_range=0.1, init_std=0.02, local_batch_size=None, local_rank=0, log_all_ranks=False, log_interval=10, lr=0.0, lr_min=0.0, max_step=400, max_step_scheduler=None, mem_len=192, mom=0.0, multi_gpu=None, n_head=8, n_layer=16, no_env=False, no_eval=True, no_test=False, not_tied=False, optim='jitlamb', patience=0, pre_lnorm=False, proj_init_std=0.01, restart='', roll=True, same_length=False, sample_softmax=-1, save_all=False, scheduler='cosine', seed=1111, swap_mem=False, target_perplexity=None, target_throughput=None, tgt_len=192, tied=True, txtlog_file='train_log.log', varlen=False, vocab='word', warmup_step=1000, weight_decay=0.0, work_dir='LM-TFM')
+world size: 1
+Collecting environment information...
+PyTorch version: 1.13.0a0+d0d6b1f
+Is debug build: False
+CUDA used to build PyTorch: 11.8
+ROCM used to build PyTorch: N/A
+
+OS: Ubuntu 20.04.5 LTS (x86_64)
+GCC version: (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0
+Clang version: Could not collect
+CMake version: version 3.22.2
+Libc version: glibc-2.31
+
+Python version: 3.8.13 | packaged by conda-forge | (default, Mar 25 2022, 06:04:10)  [GCC 10.3.0] (64-bit runtime)
+Python platform: Linux-5.19.0-46-generic-x86_64-with-glibc2.10
+Is CUDA available: True
+CUDA runtime version: 11.8.89
+GPU models and configuration: GPU 0: NVIDIA GeForce RTX 4090 Laptop GPU
+Nvidia driver version: 525.116.04
+cuDNN version: Probably one of the following:
+/usr/lib/x86_64-linux-gnu/libcudnn.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.6.0
+HIP runtime version: N/A
+MIOpen runtime version: N/A
+Is XNNPACK available: True
+
+Versions of relevant libraries:
+[pip3] functorch==0.3.0a0
+[pip3] numpy==1.22.2
+[pip3] pytorch-quantization==2.1.2
+[pip3] pytorch-transformers==1.1.0
+[pip3] torch==1.13.0a0+d0d6b1f
+[pip3] torch-tensorrt==1.3.0a0
+[pip3] torchtext==0.11.0a0
+[pip3] torchvision==0.14.0a0
+[conda] functorch                 0.3.0a0                  pypi_0    pypi
+[conda] mkl                       2020.4             h726a3e6_304    conda-forge
+[conda] mkl-include               2020.4             h726a3e6_304    conda-forge
+[conda] numpy                     1.22.2           py38h6ae9a64_0    conda-forge
+[conda] pytorch-quantization      2.1.2                    pypi_0    pypi
+[conda] pytorch-transformers      1.1.0                    pypi_0    pypi
+[conda] torch                     1.13.0a0+d0d6b1f          pypi_0    pypi
+[conda] torch-tensorrt            1.3.0a0                  pypi_0    pypi
+[conda] torchtext                 0.11.0a0                 pypi_0    pypi
+[conda] torchvision               0.14.0a0                 pypi_0    pypi
+Loading cached dataset...
+====================================================================================================
+    - work_dir : LM-TFM
+    - append_dataset : False
+    - append_time : False
+    - cuda : True
+    - fp16 : False
+    - restart : 
+    - debug : False
+    - log_all_ranks : False
+    - dllog_file : train_log.json
+    - txtlog_file : train_log.log
+    - save_all : False
+    - no_env : False
+    - no_eval : True
+    - no_test : False
+    - log_interval : 10
+    - target_throughput : None
+    - target_perplexity : None
+    - apex_amp_opt_level : O2
+    - amp : apex
+    - affinity : socket_unique_interleaved
+    - data : /data/transformer-xl/wikitext-103
+    - dataset : wt103
+    - vocab : word
+    - n_layer : 16
+    - n_head : 8
+    - d_head : 64
+    - d_embed : 512
+    - d_model : 512
+    - d_inner : 2048
+    - dropout : 0.1
+    - dropatt : 0.0
+    - pre_lnorm : False
+    - attn_type : 0
+    - not_tied : False
+    - clamp_len : -1
+    - adaptive : False
+    - div_val : 1
+    - sample_softmax : -1
+    - init : normal
+    - emb_init : normal
+    - init_range : 0.1
+    - emb_init_range : 0.01
+    - init_std : 0.02
+    - proj_init_std : 0.01
+    - optim : jitlamb
+    - lr : 0.0
+    - mom : 0.0
+    - scheduler : cosine
+    - max_step_scheduler : None
+    - warmup_step : 1000
+    - decay_rate : 0.5
+    - lr_min : 0.0
+    - clip : 0.25
+    - weight_decay : 0.0
+    - clip_nonemb : False
+    - patience : 0
+    - eta_min : 0.001
+    - max_step : 400
+    - batch_size : 8
+    - local_batch_size : None
+    - batch_chunk : 1
+    - roll : True
+    - tgt_len : 192
+    - ext_len : 0
+    - mem_len : 192
+    - seed : 1111
+    - multi_gpu : None
+    - gpu0_bsz : -1
+    - same_length : False
+    - varlen : False
+    - swap_mem : False
+    - eval_tgt_len : 192
+    - eval_batch_size : 16
+    - eval_max_steps : -1
+    - eval_interval : 5000
+    - local_rank : 0
+    - tied : True
+    - n_token : 267735
+    - n_all_param : 191948759
+    - n_nonemb_param : 54599680
+====================================================================================================
+#params = 191948759
+#non emb params = 54599680
+| epoch   1 step       10 | batches     10 / 67206 | lr 0.000e+00 | ms/batch 201.4 | tok/s    7626 | loss 12.57 | ppl 288369.39
+| epoch   1 step       20 | batches     20 / 67206 | lr 0.000e+00 | ms/batch 149.4 | tok/s   10278 | loss 12.57 | ppl 287491.60
+| epoch   1 step       30 | batches     30 / 67206 | lr 0.000e+00 | ms/batch 149.0 | tok/s   10310 | loss 12.55 | ppl 281153.25
+| epoch   1 step       40 | batches     40 / 67206 | lr 0.000e+00 | ms/batch 149.9 | tok/s   10250 | loss 12.57 | ppl 287115.41
+| epoch   1 step       50 | batches     50 / 67206 | lr 0.000e+00 | ms/batch 149.1 | tok/s   10299 | loss 12.55 | ppl 281554.17
+| epoch   1 step       60 | batches     60 / 67206 | lr 0.000e+00 | ms/batch 149.3 | tok/s   10286 | loss 12.56 | ppl 285659.90
+| epoch   1 step       70 | batches     70 / 67206 | lr 0.000e+00 | ms/batch 149.3 | tok/s   10289 | loss 12.55 | ppl 282023.76
+| epoch   1 step       80 | batches     80 / 67206 | lr 0.000e+00 | ms/batch 150.5 | tok/s   10206 | loss 12.55 | ppl 282729.15
+| epoch   1 step       90 | batches     90 / 67206 | lr 0.000e+00 | ms/batch 149.5 | tok/s   10272 | loss 12.56 | ppl 284163.99
+| epoch   1 step      100 | batches    100 / 67206 | lr 0.000e+00 | ms/batch 150.3 | tok/s   10220 | loss 12.56 | ppl 285054.04
+| epoch   1 step      110 | batches    110 / 67206 | lr 0.000e+00 | ms/batch 149.4 | tok/s   10281 | loss 12.55 | ppl 281579.25
+| epoch   1 step      120 | batches    120 / 67206 | lr 0.000e+00 | ms/batch 150.4 | tok/s   10214 | loss 12.57 | ppl 287790.75
+| epoch   1 step      130 | batches    130 / 67206 | lr 0.000e+00 | ms/batch 149.7 | tok/s   10257 | loss 12.56 | ppl 284573.87
+| epoch   1 step      140 | batches    140 / 67206 | lr 0.000e+00 | ms/batch 150.4 | tok/s   10209 | loss 12.55 | ppl 282745.63
+| epoch   1 step      150 | batches    150 / 67206 | lr 0.000e+00 | ms/batch 150.1 | tok/s   10234 | loss 12.56 | ppl 284136.70
+| epoch   1 step      160 | batches    160 / 67206 | lr 0.000e+00 | ms/batch 150.7 | tok/s   10194 | loss 12.55 | ppl 283033.84
+| epoch   1 step      170 | batches    170 / 67206 | lr 0.000e+00 | ms/batch 150.0 | tok/s   10240 | loss 12.56 | ppl 284109.28
+| epoch   1 step      180 | batches    180 / 67206 | lr 0.000e+00 | ms/batch 150.5 | tok/s   10204 | loss 12.53 | ppl 276647.14
+| epoch   1 step      190 | batches    190 / 67206 | lr 0.000e+00 | ms/batch 150.7 | tok/s   10194 | loss 12.54 | ppl 280298.77
+| epoch   1 step      200 | batches    200 / 67206 | lr 0.000e+00 | ms/batch 151.0 | tok/s   10171 | loss 12.54 | ppl 280523.08
+| epoch   1 step      210 | batches    210 / 67206 | lr 0.000e+00 | ms/batch 150.5 | tok/s   10203 | loss 12.56 | ppl 285696.43
+| epoch   1 step      220 | batches    220 / 67206 | lr 0.000e+00 | ms/batch 150.4 | tok/s   10216 | loss 12.56 | ppl 284159.54
+| epoch   1 step      230 | batches    230 / 67206 | lr 0.000e+00 | ms/batch 150.8 | tok/s   10186 | loss 12.55 | ppl 281771.48
+| epoch   1 step      240 | batches    240 / 67206 | lr 0.000e+00 | ms/batch 150.8 | tok/s   10185 | loss 12.55 | ppl 283155.57
+| epoch   1 step      250 | batches    250 / 67206 | lr 0.000e+00 | ms/batch 151.1 | tok/s   10163 | loss 12.54 | ppl 280625.85
+| epoch   1 step      260 | batches    260 / 67206 | lr 0.000e+00 | ms/batch 150.7 | tok/s   10190 | loss 12.54 | ppl 280435.48
+| epoch   1 step      270 | batches    270 / 67206 | lr 0.000e+00 | ms/batch 151.4 | tok/s   10143 | loss 12.55 | ppl 281817.68
+| epoch   1 step      280 | batches    280 / 67206 | lr 0.000e+00 | ms/batch 150.3 | tok/s   10218 | loss 12.55 | ppl 282867.13
+| epoch   1 step      290 | batches    290 / 67206 | lr 0.000e+00 | ms/batch 151.2 | tok/s   10159 | loss 12.54 | ppl 280408.17
+| epoch   1 step      300 | batches    300 / 67206 | lr 0.000e+00 | ms/batch 150.5 | tok/s   10206 | loss 12.52 | ppl 273234.24
+| epoch   1 step      310 | batches    310 / 67206 | lr 0.000e+00 | ms/batch 151.4 | tok/s   10146 | loss 12.54 | ppl 280493.73
+| epoch   1 step      320 | batches    320 / 67206 | lr 0.000e+00 | ms/batch 150.8 | tok/s   10184 | loss 12.55 | ppl 281015.12
+| epoch   1 step      330 | batches    330 / 67206 | lr 0.000e+00 | ms/batch 151.2 | tok/s   10162 | loss 12.51 | ppl 270974.58
+| epoch   1 step      340 | batches    340 / 67206 | lr 0.000e+00 | ms/batch 151.1 | tok/s   10162 | loss 12.55 | ppl 283420.63
+| epoch   1 step      350 | batches    350 / 67206 | lr 0.000e+00 | ms/batch 151.2 | tok/s   10159 | loss 12.56 | ppl 283661.73
+| epoch   1 step      360 | batches    360 / 67206 | lr 0.000e+00 | ms/batch 151.4 | tok/s   10144 | loss 12.55 | ppl 282501.33
+| epoch   1 step      370 | batches    370 / 67206 | lr 0.000e+00 | ms/batch 150.6 | tok/s   10199 | loss 12.55 | ppl 281806.18
+| epoch   1 step      380 | batches    380 / 67206 | lr 0.000e+00 | ms/batch 151.5 | tok/s   10136 | loss 12.53 | ppl 277800.29
+| epoch   1 step      390 | batches    390 / 67206 | lr 0.000e+00 | ms/batch 151.8 | tok/s   10120 | loss 12.56 | ppl 283814.34
+| epoch   1 step      400 | batches    400 / 67206 | lr 0.000e+00 | ms/batch 151.1 | tok/s   10168 | loss 12.57 | ppl 286638.66
+----------------------------------------------------------------------------------------------------
+End of training
+Training time: 1.02 minutes
+Training throughput: 10201.95 tok/s
+DONE!
diff --git a/pytorch/results/4090laptop_v1/PyTorch_transformerxlbase_FP32/benchmark.para b/pytorch/results/4090laptop_v1/PyTorch_transformerxlbase_FP32/benchmark.para
new file mode 100755
index 00000000..6ea5ca92
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_transformerxlbase_FP32/benchmark.para
@@ -0,0 +1,2 @@
+GLOBAL_BATCH 8
+GPU 1
diff --git a/pytorch/results/4090laptop_v1/PyTorch_transformerxllarge_FP16/12-07-2023_20-10-32.txt b/pytorch/results/4090laptop_v1/PyTorch_transformerxllarge_FP16/12-07-2023_20-10-32.txt
new file mode 100755
index 00000000..3a63cbc8
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_transformerxllarge_FP16/12-07-2023_20-10-32.txt
@@ -0,0 +1,202 @@
+0: thread affinity: {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}
+Experiment dir : LM-TFM
+Namespace(adaptive=False, affinity='socket_unique_interleaved', amp='apex', apex_amp_opt_level='O2', append_dataset=False, append_time=False, attn_type=0, batch_chunk=1, batch_size=4, clamp_len=-1, clip=0.25, clip_nonemb=False, cuda=True, d_embed=1024, d_head=64, d_inner=4096, d_model=1024, data='/data/transformer-xl/wikitext-103', dataset='wt103', debug=False, decay_rate=0.5, div_val=1, dllog_file='train_log.json', dropatt=0.2, dropout=0.2, emb_init='normal', emb_init_range=0.01, eta_min=0.001, eval_batch_size=16, eval_interval=5000, eval_max_steps=-1, eval_tgt_len=128, ext_len=0, fp16=True, gpu0_bsz=-1, init='normal', init_range=0.1, init_std=0.02, local_batch_size=None, local_rank=0, log_all_ranks=False, log_interval=10, lr=0.0, lr_min=0.0, max_step=400, max_step_scheduler=None, mem_len=256, mom=0.0, multi_gpu=None, n_head=16, n_layer=18, no_env=False, no_eval=True, no_test=False, not_tied=False, optim='adam', patience=0, pre_lnorm=False, proj_init_std=0.01, restart='', roll=False, same_length=False, sample_softmax=-1, save_all=False, scheduler='cosine', seed=1111, swap_mem=False, target_perplexity=None, target_throughput=None, tgt_len=256, tied=True, txtlog_file='train_log.log', varlen=False, vocab='word', warmup_step=16000, weight_decay=0.0, work_dir='LM-TFM')
+world size: 1
+Collecting environment information...
+PyTorch version: 1.13.0a0+d0d6b1f
+Is debug build: False
+CUDA used to build PyTorch: 11.8
+ROCM used to build PyTorch: N/A
+
+OS: Ubuntu 20.04.5 LTS (x86_64)
+GCC version: (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0
+Clang version: Could not collect
+CMake version: version 3.22.2
+Libc version: glibc-2.31
+
+Python version: 3.8.13 | packaged by conda-forge | (default, Mar 25 2022, 06:04:10)  [GCC 10.3.0] (64-bit runtime)
+Python platform: Linux-5.19.0-46-generic-x86_64-with-glibc2.10
+Is CUDA available: True
+CUDA runtime version: 11.8.89
+GPU models and configuration: GPU 0: NVIDIA GeForce RTX 4090 Laptop GPU
+Nvidia driver version: 525.116.04
+cuDNN version: Probably one of the following:
+/usr/lib/x86_64-linux-gnu/libcudnn.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.6.0
+HIP runtime version: N/A
+MIOpen runtime version: N/A
+Is XNNPACK available: True
+
+Versions of relevant libraries:
+[pip3] functorch==0.3.0a0
+[pip3] numpy==1.22.2
+[pip3] pytorch-quantization==2.1.2
+[pip3] pytorch-transformers==1.1.0
+[pip3] torch==1.13.0a0+d0d6b1f
+[pip3] torch-tensorrt==1.3.0a0
+[pip3] torchtext==0.11.0a0
+[pip3] torchvision==0.14.0a0
+[conda] functorch                 0.3.0a0                  pypi_0    pypi
+[conda] mkl                       2020.4             h726a3e6_304    conda-forge
+[conda] mkl-include               2020.4             h726a3e6_304    conda-forge
+[conda] numpy                     1.22.2           py38h6ae9a64_0    conda-forge
+[conda] pytorch-quantization      2.1.2                    pypi_0    pypi
+[conda] pytorch-transformers      1.1.0                    pypi_0    pypi
+[conda] torch                     1.13.0a0+d0d6b1f          pypi_0    pypi
+[conda] torch-tensorrt            1.3.0a0                  pypi_0    pypi
+[conda] torchtext                 0.11.0a0                 pypi_0    pypi
+[conda] torchvision               0.14.0a0                 pypi_0    pypi
+Loading cached dataset...
+Selected optimization level O2:  FP16 training with FP32 batchnorm and FP32 master weights.
+
+Defaults for this optimization level are:
+enabled                : True
+opt_level              : O2
+cast_model_type        : torch.float16
+patch_torch_functions  : False
+keep_batchnorm_fp32    : True
+master_weights         : True
+loss_scale             : dynamic
+Processing user overrides (additional kwargs that are not None)...
+After processing overrides, optimization options are:
+enabled                : True
+opt_level              : O2
+cast_model_type        : torch.float16
+patch_torch_functions  : False
+keep_batchnorm_fp32    : True
+master_weights         : True
+loss_scale             : dynamic
+====================================================================================================
+    - work_dir : LM-TFM
+    - append_dataset : False
+    - append_time : False
+    - cuda : True
+    - fp16 : True
+    - restart : 
+    - debug : False
+    - log_all_ranks : False
+    - dllog_file : train_log.json
+    - txtlog_file : train_log.log
+    - save_all : False
+    - no_env : False
+    - no_eval : True
+    - no_test : False
+    - log_interval : 10
+    - target_throughput : None
+    - target_perplexity : None
+    - apex_amp_opt_level : O2
+    - amp : apex
+    - affinity : socket_unique_interleaved
+    - data : /data/transformer-xl/wikitext-103
+    - dataset : wt103
+    - vocab : word
+    - n_layer : 18
+    - n_head : 16
+    - d_head : 64
+    - d_embed : 1024
+    - d_model : 1024
+    - d_inner : 4096
+    - dropout : 0.2
+    - dropatt : 0.2
+    - pre_lnorm : False
+    - attn_type : 0
+    - not_tied : False
+    - clamp_len : -1
+    - adaptive : False
+    - div_val : 1
+    - sample_softmax : -1
+    - init : normal
+    - emb_init : normal
+    - init_range : 0.1
+    - emb_init_range : 0.01
+    - init_std : 0.02
+    - proj_init_std : 0.01
+    - optim : adam
+    - lr : 0.0
+    - mom : 0.0
+    - scheduler : cosine
+    - max_step_scheduler : None
+    - warmup_step : 16000
+    - decay_rate : 0.5
+    - lr_min : 0.0
+    - clip : 0.25
+    - weight_decay : 0.0
+    - clip_nonemb : False
+    - patience : 0
+    - eta_min : 0.001
+    - max_step : 400
+    - batch_size : 4
+    - local_batch_size : None
+    - batch_chunk : 1
+    - roll : False
+    - tgt_len : 256
+    - ext_len : 0
+    - mem_len : 256
+    - seed : 1111
+    - multi_gpu : None
+    - gpu0_bsz : -1
+    - same_length : False
+    - varlen : False
+    - swap_mem : False
+    - eval_tgt_len : 128
+    - eval_batch_size : 16
+    - eval_max_steps : -1
+    - eval_interval : 5000
+    - local_rank : 0
+    - tied : True
+    - n_token : 267735
+    - n_all_param : 519963095
+    - n_nonemb_param : 245532672
+====================================================================================================
+#params = 519963095
+#non emb params = 245532672
+| epoch   1 step       10 | batches     10 / 100808 | lr 0.000e+00 | ms/batch 211.7 | tok/s    4838 | loss 12.73 | ppl 339308.21
+| epoch   1 step       20 | batches     20 / 100808 | lr 0.000e+00 | ms/batch 192.8 | tok/s    5311 | loss 12.72 | ppl 333930.00
+| epoch   1 step       30 | batches     30 / 100808 | lr 0.000e+00 | ms/batch 191.7 | tok/s    5341 | loss 12.73 | ppl 339255.15
+| epoch   1 step       40 | batches     40 / 100808 | lr 0.000e+00 | ms/batch 192.9 | tok/s    5308 | loss 12.76 | ppl 347295.85
+| epoch   1 step       50 | batches     50 / 100808 | lr 0.000e+00 | ms/batch 193.0 | tok/s    5306 | loss 12.74 | ppl 340310.49
+| epoch   1 step       60 | batches     60 / 100808 | lr 0.000e+00 | ms/batch 192.1 | tok/s    5332 | loss 12.73 | ppl 336983.01
+| epoch   1 step       70 | batches     70 / 100808 | lr 0.000e+00 | ms/batch 193.4 | tok/s    5294 | loss 12.73 | ppl 339214.77
+| epoch   1 step       80 | batches     80 / 100808 | lr 0.000e+00 | ms/batch 193.1 | tok/s    5304 | loss 12.72 | ppl 335164.34
+| epoch   1 step       90 | batches     90 / 100808 | lr 0.000e+00 | ms/batch 192.2 | tok/s    5329 | loss 12.72 | ppl 334943.99
+| epoch   1 step      100 | batches    100 / 100808 | lr 0.000e+00 | ms/batch 194.1 | tok/s    5276 | loss 12.73 | ppl 338167.22
+| epoch   1 step      110 | batches    110 / 100808 | lr 0.000e+00 | ms/batch 193.5 | tok/s    5293 | loss 12.73 | ppl 337574.59
+| epoch   1 step      120 | batches    120 / 100808 | lr 0.000e+00 | ms/batch 192.4 | tok/s    5321 | loss 12.74 | ppl 339547.75
+| epoch   1 step      130 | batches    130 / 100808 | lr 0.000e+00 | ms/batch 194.2 | tok/s    5274 | loss 12.72 | ppl 335764.26
+| epoch   1 step      140 | batches    140 / 100808 | lr 0.000e+00 | ms/batch 193.5 | tok/s    5292 | loss 12.73 | ppl 338656.48
+| epoch   1 step      150 | batches    150 / 100808 | lr 0.000e+00 | ms/batch 193.0 | tok/s    5306 | loss 12.74 | ppl 340142.54
+| epoch   1 step      160 | batches    160 / 100808 | lr 0.000e+00 | ms/batch 194.4 | tok/s    5267 | loss 12.72 | ppl 335877.50
+| epoch   1 step      170 | batches    170 / 100808 | lr 0.000e+00 | ms/batch 194.0 | tok/s    5280 | loss 12.74 | ppl 341170.19
+| epoch   1 step      180 | batches    180 / 100808 | lr 0.000e+00 | ms/batch 193.0 | tok/s    5305 | loss 12.73 | ppl 336721.90
+| epoch   1 step      190 | batches    190 / 100808 | lr 0.000e+00 | ms/batch 194.4 | tok/s    5269 | loss 12.72 | ppl 334080.35
+| epoch   1 step      200 | batches    200 / 100808 | lr 0.000e+00 | ms/batch 194.3 | tok/s    5272 | loss 12.73 | ppl 337902.61
+| epoch   1 step      210 | batches    210 / 100808 | lr 0.000e+00 | ms/batch 193.6 | tok/s    5290 | loss 12.73 | ppl 336058.47
+| epoch   1 step      220 | batches    220 / 100808 | lr 0.000e+00 | ms/batch 193.7 | tok/s    5285 | loss 12.74 | ppl 339635.58
+| epoch   1 step      230 | batches    230 / 100808 | lr 0.000e+00 | ms/batch 194.9 | tok/s    5253 | loss 12.73 | ppl 339156.03
+| epoch   1 step      240 | batches    240 / 100808 | lr 0.000e+00 | ms/batch 194.0 | tok/s    5278 | loss 12.74 | ppl 340744.88
+| epoch   1 step      250 | batches    250 / 100808 | lr 0.000e+00 | ms/batch 193.2 | tok/s    5299 | loss 12.72 | ppl 335613.15
+| epoch   1 step      260 | batches    260 / 100808 | lr 0.000e+00 | ms/batch 195.1 | tok/s    5249 | loss 12.73 | ppl 336770.71
+| epoch   1 step      270 | batches    270 / 100808 | lr 0.000e+00 | ms/batch 194.8 | tok/s    5256 | loss 12.74 | ppl 342617.09
+| epoch   1 step      280 | batches    280 / 100808 | lr 0.000e+00 | ms/batch 193.7 | tok/s    5287 | loss 12.74 | ppl 341072.59
+| epoch   1 step      290 | batches    290 / 100808 | lr 0.000e+00 | ms/batch 193.9 | tok/s    5280 | loss 12.73 | ppl 338597.58
+| epoch   1 step      300 | batches    300 / 100808 | lr 0.000e+00 | ms/batch 194.9 | tok/s    5253 | loss 12.73 | ppl 337698.24
+| epoch   1 step      310 | batches    310 / 100808 | lr 0.000e+00 | ms/batch 194.5 | tok/s    5266 | loss 12.74 | ppl 341155.35
+| epoch   1 step      320 | batches    320 / 100808 | lr 0.000e+00 | ms/batch 193.7 | tok/s    5288 | loss 12.73 | ppl 339204.42
+| epoch   1 step      330 | batches    330 / 100808 | lr 0.000e+00 | ms/batch 194.9 | tok/s    5255 | loss 12.73 | ppl 339409.45
+| epoch   1 step      340 | batches    340 / 100808 | lr 0.000e+00 | ms/batch 195.1 | tok/s    5249 | loss 12.74 | ppl 340143.84
+| epoch   1 step      350 | batches    350 / 100808 | lr 0.000e+00 | ms/batch 194.2 | tok/s    5273 | loss 12.74 | ppl 342183.19
+| epoch   1 step      360 | batches    360 / 100808 | lr 0.000e+00 | ms/batch 193.7 | tok/s    5288 | loss 12.73 | ppl 338182.18
+| epoch   1 step      370 | batches    370 / 100808 | lr 0.000e+00 | ms/batch 195.7 | tok/s    5232 | loss 12.74 | ppl 340447.86
+| epoch   1 step      380 | batches    380 / 100808 | lr 0.000e+00 | ms/batch 194.9 | tok/s    5253 | loss 12.74 | ppl 340631.29
+| epoch   1 step      390 | batches    390 / 100808 | lr 0.000e+00 | ms/batch 194.1 | tok/s    5275 | loss 12.73 | ppl 337286.00
+| epoch   1 step      400 | batches    400 / 100808 | lr 0.000e+00 | ms/batch 194.1 | tok/s    5274 | loss 12.73 | ppl 337295.78
+----------------------------------------------------------------------------------------------------
+End of training
+Training time: 1.30 minutes
+Training throughput: 5281.35 tok/s
+DONE!
diff --git a/pytorch/results/4090laptop_v1/PyTorch_transformerxllarge_FP16/benchmark.para b/pytorch/results/4090laptop_v1/PyTorch_transformerxllarge_FP16/benchmark.para
new file mode 100755
index 00000000..2b1b3f18
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_transformerxllarge_FP16/benchmark.para
@@ -0,0 +1,2 @@
+GLOBAL_BATCH 4
+GPU 1
diff --git a/pytorch/results/4090laptop_v1/PyTorch_transformerxllarge_FP32/12-07-2023_20-00-32.txt b/pytorch/results/4090laptop_v1/PyTorch_transformerxllarge_FP32/12-07-2023_20-00-32.txt
new file mode 100755
index 00000000..6f88eee9
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_transformerxllarge_FP32/12-07-2023_20-00-32.txt
@@ -0,0 +1,185 @@
+0: thread affinity: {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}
+Experiment dir : LM-TFM
+Namespace(adaptive=False, affinity='socket_unique_interleaved', amp='apex', apex_amp_opt_level='O2', append_dataset=False, append_time=False, attn_type=0, batch_chunk=1, batch_size=2, clamp_len=-1, clip=0.25, clip_nonemb=False, cuda=True, d_embed=1024, d_head=64, d_inner=4096, d_model=1024, data='/data/transformer-xl/wikitext-103', dataset='wt103', debug=False, decay_rate=0.5, div_val=1, dllog_file='train_log.json', dropatt=0.2, dropout=0.2, emb_init='normal', emb_init_range=0.01, eta_min=0.001, eval_batch_size=16, eval_interval=5000, eval_max_steps=-1, eval_tgt_len=128, ext_len=0, fp16=False, gpu0_bsz=-1, init='normal', init_range=0.1, init_std=0.02, local_batch_size=None, local_rank=0, log_all_ranks=False, log_interval=10, lr=0.0, lr_min=0.0, max_step=400, max_step_scheduler=None, mem_len=256, mom=0.0, multi_gpu=None, n_head=16, n_layer=18, no_env=False, no_eval=True, no_test=False, not_tied=False, optim='adam', patience=0, pre_lnorm=False, proj_init_std=0.01, restart='', roll=True, same_length=False, sample_softmax=-1, save_all=False, scheduler='cosine', seed=1111, swap_mem=False, target_perplexity=None, target_throughput=None, tgt_len=256, tied=True, txtlog_file='train_log.log', varlen=False, vocab='word', warmup_step=16000, weight_decay=0.0, work_dir='LM-TFM')
+world size: 1
+Collecting environment information...
+PyTorch version: 1.13.0a0+d0d6b1f
+Is debug build: False
+CUDA used to build PyTorch: 11.8
+ROCM used to build PyTorch: N/A
+
+OS: Ubuntu 20.04.5 LTS (x86_64)
+GCC version: (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0
+Clang version: Could not collect
+CMake version: version 3.22.2
+Libc version: glibc-2.31
+
+Python version: 3.8.13 | packaged by conda-forge | (default, Mar 25 2022, 06:04:10)  [GCC 10.3.0] (64-bit runtime)
+Python platform: Linux-5.19.0-46-generic-x86_64-with-glibc2.10
+Is CUDA available: True
+CUDA runtime version: 11.8.89
+GPU models and configuration: GPU 0: NVIDIA GeForce RTX 4090 Laptop GPU
+Nvidia driver version: 525.116.04
+cuDNN version: Probably one of the following:
+/usr/lib/x86_64-linux-gnu/libcudnn.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.6.0
+/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.6.0
+HIP runtime version: N/A
+MIOpen runtime version: N/A
+Is XNNPACK available: True
+
+Versions of relevant libraries:
+[pip3] functorch==0.3.0a0
+[pip3] numpy==1.22.2
+[pip3] pytorch-quantization==2.1.2
+[pip3] pytorch-transformers==1.1.0
+[pip3] torch==1.13.0a0+d0d6b1f
+[pip3] torch-tensorrt==1.3.0a0
+[pip3] torchtext==0.11.0a0
+[pip3] torchvision==0.14.0a0
+[conda] functorch                 0.3.0a0                  pypi_0    pypi
+[conda] mkl                       2020.4             h726a3e6_304    conda-forge
+[conda] mkl-include               2020.4             h726a3e6_304    conda-forge
+[conda] numpy                     1.22.2           py38h6ae9a64_0    conda-forge
+[conda] pytorch-quantization      2.1.2                    pypi_0    pypi
+[conda] pytorch-transformers      1.1.0                    pypi_0    pypi
+[conda] torch                     1.13.0a0+d0d6b1f          pypi_0    pypi
+[conda] torch-tensorrt            1.3.0a0                  pypi_0    pypi
+[conda] torchtext                 0.11.0a0                 pypi_0    pypi
+[conda] torchvision               0.14.0a0                 pypi_0    pypi
+Producing dataset wt103...
+building vocab with min_freq=0, max_size=None
+final vocab size 267735 from 267734 unique tokens
+====================================================================================================
+    - work_dir : LM-TFM
+    - append_dataset : False
+    - append_time : False
+    - cuda : True
+    - fp16 : False
+    - restart : 
+    - debug : False
+    - log_all_ranks : False
+    - dllog_file : train_log.json
+    - txtlog_file : train_log.log
+    - save_all : False
+    - no_env : False
+    - no_eval : True
+    - no_test : False
+    - log_interval : 10
+    - target_throughput : None
+    - target_perplexity : None
+    - apex_amp_opt_level : O2
+    - amp : apex
+    - affinity : socket_unique_interleaved
+    - data : /data/transformer-xl/wikitext-103
+    - dataset : wt103
+    - vocab : word
+    - n_layer : 18
+    - n_head : 16
+    - d_head : 64
+    - d_embed : 1024
+    - d_model : 1024
+    - d_inner : 4096
+    - dropout : 0.2
+    - dropatt : 0.2
+    - pre_lnorm : False
+    - attn_type : 0
+    - not_tied : False
+    - clamp_len : -1
+    - adaptive : False
+    - div_val : 1
+    - sample_softmax : -1
+    - init : normal
+    - emb_init : normal
+    - init_range : 0.1
+    - emb_init_range : 0.01
+    - init_std : 0.02
+    - proj_init_std : 0.01
+    - optim : adam
+    - lr : 0.0
+    - mom : 0.0
+    - scheduler : cosine
+    - max_step_scheduler : None
+    - warmup_step : 16000
+    - decay_rate : 0.5
+    - lr_min : 0.0
+    - clip : 0.25
+    - weight_decay : 0.0
+    - clip_nonemb : False
+    - patience : 0
+    - eta_min : 0.001
+    - max_step : 400
+    - batch_size : 2
+    - local_batch_size : None
+    - batch_chunk : 1
+    - roll : True
+    - tgt_len : 256
+    - ext_len : 0
+    - mem_len : 256
+    - seed : 1111
+    - multi_gpu : None
+    - gpu0_bsz : -1
+    - same_length : False
+    - varlen : False
+    - swap_mem : False
+    - eval_tgt_len : 128
+    - eval_batch_size : 16
+    - eval_max_steps : -1
+    - eval_interval : 5000
+    - local_rank : 0
+    - tied : True
+    - n_token : 267735
+    - n_all_param : 519963095
+    - n_nonemb_param : 245532672
+====================================================================================================
+#params = 519963095
+#non emb params = 245532672
+| epoch   1 step       10 | batches     10 / 201616 | lr 0.000e+00 | ms/batch 194.3 | tok/s    2636 | loss 12.75 | ppl 345086.14
+| epoch   1 step       20 | batches     20 / 201616 | lr 0.000e+00 | ms/batch 177.7 | tok/s    2882 | loss 12.72 | ppl 334063.56
+| epoch   1 step       30 | batches     30 / 201616 | lr 0.000e+00 | ms/batch 178.2 | tok/s    2873 | loss 12.72 | ppl 335614.66
+| epoch   1 step       40 | batches     40 / 201616 | lr 0.000e+00 | ms/batch 178.2 | tok/s    2873 | loss 12.74 | ppl 339661.63
+| epoch   1 step       50 | batches     50 / 201616 | lr 0.000e+00 | ms/batch 178.5 | tok/s    2868 | loss 12.74 | ppl 339706.36
+| epoch   1 step       60 | batches     60 / 201616 | lr 0.000e+00 | ms/batch 178.4 | tok/s    2870 | loss 12.75 | ppl 343706.65
+| epoch   1 step       70 | batches     70 / 201616 | lr 0.000e+00 | ms/batch 178.5 | tok/s    2868 | loss 12.73 | ppl 338551.21
+| epoch   1 step       80 | batches     80 / 201616 | lr 0.000e+00 | ms/batch 178.4 | tok/s    2869 | loss 12.74 | ppl 342223.23
+| epoch   1 step       90 | batches     90 / 201616 | lr 0.000e+00 | ms/batch 178.8 | tok/s    2864 | loss 12.74 | ppl 339441.40
+| epoch   1 step      100 | batches    100 / 201616 | lr 0.000e+00 | ms/batch 179.1 | tok/s    2858 | loss 12.75 | ppl 343591.55
+| epoch   1 step      110 | batches    110 / 201616 | lr 0.000e+00 | ms/batch 179.0 | tok/s    2860 | loss 12.73 | ppl 336788.09
+| epoch   1 step      120 | batches    120 / 201616 | lr 0.000e+00 | ms/batch 179.4 | tok/s    2854 | loss 12.73 | ppl 336731.02
+| epoch   1 step      130 | batches    130 / 201616 | lr 0.000e+00 | ms/batch 179.5 | tok/s    2853 | loss 12.74 | ppl 340834.38
+| epoch   1 step      140 | batches    140 / 201616 | lr 0.000e+00 | ms/batch 179.5 | tok/s    2852 | loss 12.75 | ppl 343298.67
+| epoch   1 step      150 | batches    150 / 201616 | lr 0.000e+00 | ms/batch 179.9 | tok/s    2846 | loss 12.75 | ppl 343892.75
+| epoch   1 step      160 | batches    160 / 201616 | lr 0.000e+00 | ms/batch 179.9 | tok/s    2847 | loss 12.74 | ppl 340759.08
+| epoch   1 step      170 | batches    170 / 201616 | lr 0.000e+00 | ms/batch 179.8 | tok/s    2848 | loss 12.74 | ppl 341738.49
+| epoch   1 step      180 | batches    180 / 201616 | lr 0.000e+00 | ms/batch 180.3 | tok/s    2839 | loss 12.74 | ppl 341768.80
+| epoch   1 step      190 | batches    190 / 201616 | lr 0.000e+00 | ms/batch 179.6 | tok/s    2851 | loss 12.73 | ppl 337403.08
+| epoch   1 step      200 | batches    200 / 201616 | lr 0.000e+00 | ms/batch 180.1 | tok/s    2843 | loss 12.69 | ppl 325441.76
+| epoch   1 step      210 | batches    210 / 201616 | lr 0.000e+00 | ms/batch 180.4 | tok/s    2839 | loss 12.70 | ppl 328531.94
+| epoch   1 step      220 | batches    220 / 201616 | lr 0.000e+00 | ms/batch 180.2 | tok/s    2841 | loss 12.72 | ppl 332823.53
+| epoch   1 step      230 | batches    230 / 201616 | lr 0.000e+00 | ms/batch 180.2 | tok/s    2841 | loss 12.70 | ppl 326829.31
+| epoch   1 step      240 | batches    240 / 201616 | lr 0.000e+00 | ms/batch 180.2 | tok/s    2841 | loss 12.73 | ppl 336925.94
+| epoch   1 step      250 | batches    250 / 201616 | lr 0.000e+00 | ms/batch 180.5 | tok/s    2836 | loss 12.74 | ppl 342794.17
+| epoch   1 step      260 | batches    260 / 201616 | lr 0.000e+00 | ms/batch 180.7 | tok/s    2834 | loss 12.75 | ppl 343022.39
+| epoch   1 step      270 | batches    270 / 201616 | lr 0.000e+00 | ms/batch 180.6 | tok/s    2835 | loss 12.73 | ppl 336486.88
+| epoch   1 step      280 | batches    280 / 201616 | lr 0.000e+00 | ms/batch 180.6 | tok/s    2835 | loss 12.71 | ppl 331725.60
+| epoch   1 step      290 | batches    290 / 201616 | lr 0.000e+00 | ms/batch 180.8 | tok/s    2832 | loss 12.74 | ppl 342299.77
+| epoch   1 step      300 | batches    300 / 201616 | lr 0.000e+00 | ms/batch 180.5 | tok/s    2836 | loss 12.72 | ppl 335522.65
+| epoch   1 step      310 | batches    310 / 201616 | lr 0.000e+00 | ms/batch 180.8 | tok/s    2831 | loss 12.71 | ppl 332666.96
+| epoch   1 step      320 | batches    320 / 201616 | lr 0.000e+00 | ms/batch 180.8 | tok/s    2832 | loss 12.74 | ppl 341001.46
+| epoch   1 step      330 | batches    330 / 201616 | lr 0.000e+00 | ms/batch 180.9 | tok/s    2831 | loss 12.73 | ppl 336161.84
+| epoch   1 step      340 | batches    340 / 201616 | lr 0.000e+00 | ms/batch 180.9 | tok/s    2831 | loss 12.72 | ppl 335848.71
+| epoch   1 step      350 | batches    350 / 201616 | lr 0.000e+00 | ms/batch 180.8 | tok/s    2832 | loss 12.74 | ppl 342003.46
+| epoch   1 step      360 | batches    360 / 201616 | lr 0.000e+00 | ms/batch 180.9 | tok/s    2830 | loss 12.72 | ppl 334138.63
+| epoch   1 step      370 | batches    370 / 201616 | lr 0.000e+00 | ms/batch 180.8 | tok/s    2831 | loss 12.73 | ppl 338256.95
+| epoch   1 step      380 | batches    380 / 201616 | lr 0.000e+00 | ms/batch 181.2 | tok/s    2825 | loss 12.72 | ppl 334066.90
+| epoch   1 step      390 | batches    390 / 201616 | lr 0.000e+00 | ms/batch 180.7 | tok/s    2833 | loss 12.75 | ppl 344612.53
+| epoch   1 step      400 | batches    400 / 201616 | lr 0.000e+00 | ms/batch 181.2 | tok/s    2825 | loss 12.73 | ppl 338459.53
+----------------------------------------------------------------------------------------------------
+End of training
+Training time: 1.20 minutes
+Training throughput: 2844.15 tok/s
+DONE!
diff --git a/pytorch/results/4090laptop_v1/PyTorch_transformerxllarge_FP32/benchmark.para b/pytorch/results/4090laptop_v1/PyTorch_transformerxllarge_FP32/benchmark.para
new file mode 100755
index 00000000..28ccd90a
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_transformerxllarge_FP32/benchmark.para
@@ -0,0 +1,2 @@
+GLOBAL_BATCH 2
+GPU 1
diff --git a/pytorch/results/4090laptop_v1/PyTorch_waveglow_FP16/12-07-2023_19-58-10.txt b/pytorch/results/4090laptop_v1/PyTorch_waveglow_FP16/12-07-2023_19-58-10.txt
new file mode 100755
index 00000000..9446f12f
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_waveglow_FP16/12-07-2023_19-58-10.txt
@@ -0,0 +1,709 @@
+OMP: Info #276: omp_set_nested routine deprecated, please use omp_set_max_active_levels instead.
+DLL 2023-07-12 19:58:17.890379 - PARAMETER output : ./ 
+DLL 2023-07-12 19:58:17.890410 - PARAMETER dataset_path : /data/tacotron2/LJSpeech-1.1 
+DLL 2023-07-12 19:58:17.890421 - PARAMETER model_name : WaveGlow 
+DLL 2023-07-12 19:58:17.890429 - PARAMETER log_file : nvlog.json 
+DLL 2023-07-12 19:58:17.890436 - PARAMETER anneal_steps : None 
+DLL 2023-07-12 19:58:17.890442 - PARAMETER anneal_factor : 0.1 
+DLL 2023-07-12 19:58:17.890449 - PARAMETER config_file : None 
+DLL 2023-07-12 19:58:17.890456 - PARAMETER seed : None 
+DLL 2023-07-12 19:58:17.890462 - PARAMETER epochs : 2 
+DLL 2023-07-12 19:58:17.890468 - PARAMETER epochs_per_checkpoint : 50 
+DLL 2023-07-12 19:58:17.890475 - PARAMETER checkpoint_path :  
+DLL 2023-07-12 19:58:17.890481 - PARAMETER resume_from_last : False 
+DLL 2023-07-12 19:58:17.890488 - PARAMETER dynamic_loss_scaling : True 
+DLL 2023-07-12 19:58:17.890494 - PARAMETER amp : False 
+DLL 2023-07-12 19:58:17.890500 - PARAMETER cudnn_enabled : True 
+DLL 2023-07-12 19:58:17.890506 - PARAMETER cudnn_benchmark : True 
+DLL 2023-07-12 19:58:17.890512 - PARAMETER disable_uniform_initialize_bn_weight : False 
+DLL 2023-07-12 19:58:17.890518 - PARAMETER use_saved_learning_rate : False 
+DLL 2023-07-12 19:58:17.890524 - PARAMETER learning_rate : 0.0 
+DLL 2023-07-12 19:58:17.890530 - PARAMETER weight_decay : 0.0 
+DLL 2023-07-12 19:58:17.890536 - PARAMETER grad_clip_thresh : 65504.0 
+DLL 2023-07-12 19:58:17.890542 - PARAMETER batch_size : 8 
+DLL 2023-07-12 19:58:17.890549 - PARAMETER grad_clip : 5.0 
+DLL 2023-07-12 19:58:17.890555 - PARAMETER load_mel_from_disk : False 
+DLL 2023-07-12 19:58:17.890561 - PARAMETER training_files : filelists/ljs_audio_text_train_subset_625_filelist.txt 
+DLL 2023-07-12 19:58:17.890567 - PARAMETER validation_files : filelists/ljs_audio_text_val_filelist.txt 
+DLL 2023-07-12 19:58:17.890573 - PARAMETER text_cleaners : ['english_cleaners'] 
+DLL 2023-07-12 19:58:17.890581 - PARAMETER max_wav_value : 32768.0 
+DLL 2023-07-12 19:58:17.890587 - PARAMETER sampling_rate : 22050 
+DLL 2023-07-12 19:58:17.890593 - PARAMETER filter_length : 1024 
+DLL 2023-07-12 19:58:17.890600 - PARAMETER hop_length : 256 
+DLL 2023-07-12 19:58:17.890606 - PARAMETER win_length : 1024 
+DLL 2023-07-12 19:58:17.890612 - PARAMETER mel_fmin : 0.0 
+DLL 2023-07-12 19:58:17.890618 - PARAMETER mel_fmax : 8000.0 
+DLL 2023-07-12 19:58:17.890624 - PARAMETER rank : 0 
+DLL 2023-07-12 19:58:17.890630 - PARAMETER world_size : 1 
+DLL 2023-07-12 19:58:17.890636 - PARAMETER dist_url : tcp://localhost:23456 
+DLL 2023-07-12 19:58:17.890642 - PARAMETER group_name : group_name 
+DLL 2023-07-12 19:58:17.890647 - PARAMETER dist_backend : nccl 
+DLL 2023-07-12 19:58:17.890654 - PARAMETER bench_class :  
+DLL 2023-07-12 19:58:17.890659 - PARAMETER model_name : Tacotron2_PyT 
+DLL 2023-07-12 19:58:22.527981 - (0, 0) glob_iter/iters_per_epoch : 0/78 
+DLL 2023-07-12 19:58:24.619549 - (0, 0) train_loss : 0.0027315898332744837 
+DLL 2023-07-12 19:58:25.867253 - (0, 0) train_items_per_sec : 19165.655763922587 items/s
+DLL 2023-07-12 19:58:25.867299 - (0, 0) train_iter_time : 3.3393065589998514 
+DLL 2023-07-12 19:58:25.867427 - (0, 1) glob_iter/iters_per_epoch : 1/78 
+DLL 2023-07-12 19:58:26.070635 - (0, 1) train_loss : 0.0016195639036595821 
+DLL 2023-07-12 19:58:26.576787 - (0, 1) train_items_per_sec : 90221.54392100463 items/s
+DLL 2023-07-12 19:58:26.576819 - (0, 1) train_iter_time : 0.7093649389998973 
+DLL 2023-07-12 19:58:26.576924 - (0, 2) glob_iter/iters_per_epoch : 2/78 
+DLL 2023-07-12 19:58:26.786862 - (0, 2) train_loss : 0.0018263020319864154 
+DLL 2023-07-12 19:58:27.295610 - (0, 2) train_items_per_sec : 89050.88630340698 items/s
+DLL 2023-07-12 19:58:27.295641 - (0, 2) train_iter_time : 0.7186902080002255 
+DLL 2023-07-12 19:58:27.295746 - (0, 3) glob_iter/iters_per_epoch : 3/78 
+DLL 2023-07-12 19:58:27.500776 - (0, 3) train_loss : 0.00204037269577384 
+DLL 2023-07-12 19:58:28.011399 - (0, 3) train_items_per_sec : 89428.47122001126 items/s
+DLL 2023-07-12 19:58:28.011434 - (0, 3) train_iter_time : 0.7156557539997266 
+DLL 2023-07-12 19:58:28.011541 - (0, 4) glob_iter/iters_per_epoch : 4/78 
+DLL 2023-07-12 19:58:28.221289 - (0, 4) train_loss : 0.0015693346504122019 
+DLL 2023-07-12 19:58:28.728992 - (0, 4) train_items_per_sec : 89204.23502790801 items/s
+DLL 2023-07-12 19:58:28.729022 - (0, 4) train_iter_time : 0.7174547260001418 
+DLL 2023-07-12 19:58:28.734106 - (0, 5) glob_iter/iters_per_epoch : 5/78 
+DLL 2023-07-12 19:58:28.938111 - (0, 5) train_loss : 0.0016553705791011453 
+DLL 2023-07-12 19:58:29.449241 - (0, 5) train_items_per_sec : 89492.8868229075 items/s
+DLL 2023-07-12 19:58:29.449274 - (0, 5) train_iter_time : 0.7151406359998873 
+DLL 2023-07-12 19:58:29.449378 - (0, 6) glob_iter/iters_per_epoch : 6/78 
+DLL 2023-07-12 19:58:29.659520 - (0, 6) train_loss : 0.002358214696869254 
+DLL 2023-07-12 19:58:30.169653 - (0, 6) train_items_per_sec : 88854.47900466382 items/s
+DLL 2023-07-12 19:58:30.169683 - (0, 6) train_iter_time : 0.7202788279996639 
+DLL 2023-07-12 19:58:30.169789 - (0, 7) glob_iter/iters_per_epoch : 7/78 
+DLL 2023-07-12 19:58:30.380327 - (0, 7) train_loss : 0.006040952168405056 
+DLL 2023-07-12 19:58:30.890128 - (0, 7) train_items_per_sec : 88846.59914164155 items/s
+DLL 2023-07-12 19:58:30.890158 - (0, 7) train_iter_time : 0.7203427100002955 
+DLL 2023-07-12 19:58:30.890262 - (0, 8) glob_iter/iters_per_epoch : 8/78 
+DLL 2023-07-12 19:58:31.098339 - (0, 8) train_loss : 0.0020325970835983753 
+DLL 2023-07-12 19:58:31.609539 - (0, 8) train_items_per_sec : 88977.65979922879 items/s
+DLL 2023-07-12 19:58:31.609569 - (0, 8) train_iter_time : 0.7192816729998412 
+DLL 2023-07-12 19:58:31.609672 - (0, 9) glob_iter/iters_per_epoch : 9/78 
+DLL 2023-07-12 19:58:31.820468 - (0, 9) train_loss : 0.0022731295321136713 
+DLL 2023-07-12 19:58:32.329402 - (0, 9) train_items_per_sec : 88921.82244436152 items/s
+DLL 2023-07-12 19:58:32.329433 - (0, 9) train_iter_time : 0.7197333369999797 
+DLL 2023-07-12 19:58:32.329532 - (0, 10) glob_iter/iters_per_epoch : 10/78 
+DLL 2023-07-12 19:58:32.534613 - (0, 10) train_loss : 0.0012079558800905943 
+DLL 2023-07-12 19:58:33.047992 - (0, 10) train_items_per_sec : 89078.97438858479 items/s
+DLL 2023-07-12 19:58:33.048022 - (0, 10) train_iter_time : 0.7184635930002514 
+DLL 2023-07-12 19:58:33.048152 - (0, 11) glob_iter/iters_per_epoch : 11/78 
+DLL 2023-07-12 19:58:33.258083 - (0, 11) train_loss : 0.0027061288710683584 
+DLL 2023-07-12 19:58:33.768300 - (0, 11) train_items_per_sec : 88870.17492839109 items/s
+DLL 2023-07-12 19:58:33.768337 - (0, 11) train_iter_time : 0.7201516149998497 
+DLL 2023-07-12 19:58:33.768440 - (0, 12) glob_iter/iters_per_epoch : 12/78 
+DLL 2023-07-12 19:58:33.980894 - (0, 12) train_loss : 0.0014180203434079885 
+DLL 2023-07-12 19:58:34.491576 - (0, 12) train_items_per_sec : 88502.8907830595 items/s
+DLL 2023-07-12 19:58:34.491606 - (0, 12) train_iter_time : 0.7231402209999942 
+DLL 2023-07-12 19:58:34.493022 - (0, 13) glob_iter/iters_per_epoch : 13/78 
+DLL 2023-07-12 19:58:34.699514 - (0, 13) train_loss : 0.0017097407253459096 
+DLL 2023-07-12 19:58:35.212134 - (0, 13) train_items_per_sec : 88997.87345978987 items/s
+DLL 2023-07-12 19:58:35.212164 - (0, 13) train_iter_time : 0.7191183059999275 
+DLL 2023-07-12 19:58:35.212265 - (0, 14) glob_iter/iters_per_epoch : 14/78 
+DLL 2023-07-12 19:58:35.422711 - (0, 14) train_loss : 0.003117230022326112 
+DLL 2023-07-12 19:58:35.932928 - (0, 14) train_items_per_sec : 88806.96162985773 items/s
+DLL 2023-07-12 19:58:35.932968 - (0, 14) train_iter_time : 0.7206642230003126 
+DLL 2023-07-12 19:58:35.933084 - (0, 15) glob_iter/iters_per_epoch : 15/78 
+DLL 2023-07-12 19:58:36.129261 - (0, 15) train_loss : 0.0019354213727638125 
+DLL 2023-07-12 19:58:36.647203 - (0, 15) train_items_per_sec : 89620.19942502982 items/s
+DLL 2023-07-12 19:58:36.647234 - (0, 15) train_iter_time : 0.7141247220001787 
+DLL 2023-07-12 19:58:36.647342 - (0, 16) glob_iter/iters_per_epoch : 16/78 
+DLL 2023-07-12 19:58:36.857621 - (0, 16) train_loss : 0.0016510533168911934 
+DLL 2023-07-12 19:58:37.369114 - (0, 16) train_items_per_sec : 88670.07736954305 items/s
+DLL 2023-07-12 19:58:37.369144 - (0, 16) train_iter_time : 0.7217767469996943 
+DLL 2023-07-12 19:58:37.370423 - (0, 17) glob_iter/iters_per_epoch : 17/78 
+DLL 2023-07-12 19:58:37.581834 - (0, 17) train_loss : 0.002353820949792862 
+DLL 2023-07-12 19:58:38.092620 - (0, 17) train_items_per_sec : 88617.94726916963 items/s
+DLL 2023-07-12 19:58:38.092656 - (0, 17) train_iter_time : 0.722201337000115 
+DLL 2023-07-12 19:58:38.094035 - (0, 18) glob_iter/iters_per_epoch : 18/78 
+DLL 2023-07-12 19:58:38.299980 - (0, 18) train_loss : 0.0013930763816460967 
+DLL 2023-07-12 19:58:38.814311 - (0, 18) train_items_per_sec : 88854.32085584362 items/s
+DLL 2023-07-12 19:58:38.814341 - (0, 18) train_iter_time : 0.7202801099997487 
+DLL 2023-07-12 19:58:38.815427 - (0, 19) glob_iter/iters_per_epoch : 19/78 
+DLL 2023-07-12 19:58:39.025614 - (0, 19) train_loss : 0.0018532362300902605 
+DLL 2023-07-12 19:58:39.535697 - (0, 19) train_items_per_sec : 88855.0868095475 items/s
+DLL 2023-07-12 19:58:39.535726 - (0, 19) train_iter_time : 0.7202739009999277 
+DLL 2023-07-12 19:58:39.537053 - (0, 20) glob_iter/iters_per_epoch : 20/78 
+DLL 2023-07-12 19:58:39.744166 - (0, 20) train_loss : 0.0012267675483599305 
+DLL 2023-07-12 19:58:40.258029 - (0, 20) train_items_per_sec : 88767.79543777418 items/s
+DLL 2023-07-12 19:58:40.258060 - (0, 20) train_iter_time : 0.7209821949995785 
+DLL 2023-07-12 19:58:40.259297 - (0, 21) glob_iter/iters_per_epoch : 21/78 
+DLL 2023-07-12 19:58:40.469431 - (0, 21) train_loss : 0.0024546729400753975 
+DLL 2023-07-12 19:58:40.982322 - (0, 21) train_items_per_sec : 88516.30605346397 items/s
+DLL 2023-07-12 19:58:40.982353 - (0, 21) train_iter_time : 0.7230306239998754 
+DLL 2023-07-12 19:58:40.983490 - (0, 22) glob_iter/iters_per_epoch : 22/78 
+DLL 2023-07-12 19:58:41.194696 - (0, 22) train_loss : 0.0017696942668408155 
+DLL 2023-07-12 19:58:41.705929 - (0, 22) train_items_per_sec : 88588.01740537939 items/s
+DLL 2023-07-12 19:58:41.705959 - (0, 22) train_iter_time : 0.7224453359999643 
+DLL 2023-07-12 19:58:41.707262 - (0, 23) glob_iter/iters_per_epoch : 23/78 
+DLL 2023-07-12 19:58:41.909882 - (0, 23) train_loss : 0.002493698848411441 
+DLL 2023-07-12 19:58:42.426340 - (0, 23) train_items_per_sec : 89002.20860235544 items/s
+DLL 2023-07-12 19:58:42.426370 - (0, 23) train_iter_time : 0.719083278999733 
+DLL 2023-07-12 19:58:42.427575 - (0, 24) glob_iter/iters_per_epoch : 24/78 
+DLL 2023-07-12 19:58:42.638525 - (0, 24) train_loss : 0.001345318858511746 
+DLL 2023-07-12 19:58:43.149896 - (0, 24) train_items_per_sec : 88602.59434016918 items/s
+DLL 2023-07-12 19:58:43.149927 - (0, 24) train_iter_time : 0.722326479000003 
+DLL 2023-07-12 19:58:43.151164 - (0, 25) glob_iter/iters_per_epoch : 25/78 
+DLL 2023-07-12 19:58:43.361077 - (0, 25) train_loss : 0.0027049905620515347 
+DLL 2023-07-12 19:58:43.876029 - (0, 25) train_items_per_sec : 88291.51341423436 items/s
+DLL 2023-07-12 19:58:43.876076 - (0, 25) train_iter_time : 0.7248714799998197 
+DLL 2023-07-12 19:58:43.877796 - (0, 26) glob_iter/iters_per_epoch : 26/78 
+DLL 2023-07-12 19:58:44.087986 - (0, 26) train_loss : 0.00335557758808136 
+DLL 2023-07-12 19:58:44.600303 - (0, 26) train_items_per_sec : 88579.7888544766 items/s
+DLL 2023-07-12 19:58:44.600338 - (0, 26) train_iter_time : 0.7225124470000992 
+DLL 2023-07-12 19:58:44.601770 - (0, 27) glob_iter/iters_per_epoch : 27/78 
+DLL 2023-07-12 19:58:44.812353 - (0, 27) train_loss : 0.003046278143301606 
+DLL 2023-07-12 19:58:45.325752 - (0, 27) train_items_per_sec : 88399.42397943296 items/s
+DLL 2023-07-12 19:58:45.325783 - (0, 27) train_iter_time : 0.7239866179997989 
+DLL 2023-07-12 19:58:45.327010 - (0, 28) glob_iter/iters_per_epoch : 28/78 
+DLL 2023-07-12 19:58:45.532569 - (0, 28) train_loss : 0.0014434014447033405 
+DLL 2023-07-12 19:58:46.046777 - (0, 28) train_items_per_sec : 88917.26349914967 items/s
+DLL 2023-07-12 19:58:46.046818 - (0, 28) train_iter_time : 0.7197702389998994 
+DLL 2023-07-12 19:58:46.047918 - (0, 29) glob_iter/iters_per_epoch : 29/78 
+DLL 2023-07-12 19:58:46.258362 - (0, 29) train_loss : 0.0021681352518498898 
+DLL 2023-07-12 19:58:46.771579 - (0, 29) train_items_per_sec : 88438.45720943116 items/s
+DLL 2023-07-12 19:58:46.771609 - (0, 29) train_iter_time : 0.7236670789998243 
+DLL 2023-07-12 19:58:46.773072 - (0, 30) glob_iter/iters_per_epoch : 30/78 
+DLL 2023-07-12 19:58:46.984657 - (0, 30) train_loss : 0.0011887593427672982 
+DLL 2023-07-12 19:58:47.496168 - (0, 30) train_items_per_sec : 88507.6232480132 items/s
+DLL 2023-07-12 19:58:47.496202 - (0, 30) train_iter_time : 0.7231015550000848 
+DLL 2023-07-12 19:58:47.497460 - (0, 31) glob_iter/iters_per_epoch : 31/78 
+DLL 2023-07-12 19:58:47.701873 - (0, 31) train_loss : 0.001460683299228549 
+DLL 2023-07-12 19:58:48.219912 - (0, 31) train_items_per_sec : 88586.5771054471 items/s
+DLL 2023-07-12 19:58:48.219946 - (0, 31) train_iter_time : 0.7224570820003464 
+DLL 2023-07-12 19:58:48.221339 - (0, 32) glob_iter/iters_per_epoch : 32/78 
+DLL 2023-07-12 19:58:48.435747 - (0, 32) train_loss : 0.0019110097782686353 
+DLL 2023-07-12 19:58:48.946475 - (0, 32) train_items_per_sec : 88258.4884159625 items/s
+DLL 2023-07-12 19:58:48.946506 - (0, 32) train_iter_time : 0.7251427159999366 
+DLL 2023-07-12 19:58:48.947617 - (0, 33) glob_iter/iters_per_epoch : 33/78 
+DLL 2023-07-12 19:58:49.157619 - (0, 33) train_loss : 0.002286675153300166 
+DLL 2023-07-12 19:58:49.674413 - (0, 33) train_items_per_sec : 88057.02433105448 items/s
+DLL 2023-07-12 19:58:49.674447 - (0, 33) train_iter_time : 0.7268017569999756 
+DLL 2023-07-12 19:58:49.675619 - (0, 34) glob_iter/iters_per_epoch : 34/78 
+DLL 2023-07-12 19:58:49.885041 - (0, 34) train_loss : 0.003575184615328908 
+DLL 2023-07-12 19:58:50.397812 - (0, 34) train_items_per_sec : 88618.33563323638 items/s
+DLL 2023-07-12 19:58:50.397844 - (0, 34) train_iter_time : 0.7221981719999349 
+DLL 2023-07-12 19:58:50.399309 - (0, 35) glob_iter/iters_per_epoch : 35/78 
+DLL 2023-07-12 19:58:50.610660 - (0, 35) train_loss : 0.0015959016745910048 
+DLL 2023-07-12 19:58:51.124852 - (0, 35) train_items_per_sec : 88209.15806030396 items/s
+DLL 2023-07-12 19:58:51.124884 - (0, 35) train_iter_time : 0.7255482470000061 
+DLL 2023-07-12 19:58:51.126053 - (0, 36) glob_iter/iters_per_epoch : 36/78 
+DLL 2023-07-12 19:58:51.332560 - (0, 36) train_loss : 0.0032364875078201294 
+DLL 2023-07-12 19:58:51.847225 - (0, 36) train_items_per_sec : 88743.76214720843 items/s
+DLL 2023-07-12 19:58:51.847257 - (0, 36) train_iter_time : 0.7211774490001517 
+DLL 2023-07-12 19:58:51.848839 - (0, 37) glob_iter/iters_per_epoch : 37/78 
+DLL 2023-07-12 19:58:52.059169 - (0, 37) train_loss : 0.0013969828141853213 
+DLL 2023-07-12 19:58:52.573273 - (0, 37) train_items_per_sec : 88344.13453796583 items/s
+DLL 2023-07-12 19:58:52.573302 - (0, 37) train_iter_time : 0.7244397190002019 
+DLL 2023-07-12 19:58:52.574547 - (0, 38) glob_iter/iters_per_epoch : 38/78 
+DLL 2023-07-12 19:58:52.786589 - (0, 38) train_loss : 0.0019526515388861299 
+DLL 2023-07-12 19:58:53.300034 - (0, 38) train_items_per_sec : 88216.07651292923 items/s
+DLL 2023-07-12 19:58:53.300081 - (0, 38) train_iter_time : 0.7254913450001368 
+DLL 2023-07-12 19:58:53.301285 - (0, 39) glob_iter/iters_per_epoch : 39/78 
+DLL 2023-07-12 19:58:53.506669 - (0, 39) train_loss : 0.001608281396329403 
+DLL 2023-07-12 19:58:54.022657 - (0, 39) train_items_per_sec : 88719.20355441274 items/s
+DLL 2023-07-12 19:58:54.022688 - (0, 39) train_iter_time : 0.7213770799999111 
+DLL 2023-07-12 19:58:54.023766 - (0, 40) glob_iter/iters_per_epoch : 40/78 
+DLL 2023-07-12 19:58:54.235272 - (0, 40) train_loss : 0.0022298411931842566 
+DLL 2023-07-12 19:58:54.749942 - (0, 40) train_items_per_sec : 88132.00777870398 items/s
+DLL 2023-07-12 19:58:54.750002 - (0, 40) train_iter_time : 0.7261833880002087 
+DLL 2023-07-12 19:58:54.751283 - (0, 41) glob_iter/iters_per_epoch : 41/78 
+DLL 2023-07-12 19:58:54.962166 - (0, 41) train_loss : 0.0019120629876852036 
+DLL 2023-07-12 19:58:55.475755 - (0, 41) train_items_per_sec : 88339.57878744071 items/s
+DLL 2023-07-12 19:58:55.475788 - (0, 41) train_iter_time : 0.7244770789998256 
+DLL 2023-07-12 19:58:55.477321 - (0, 42) glob_iter/iters_per_epoch : 42/78 
+DLL 2023-07-12 19:58:55.688393 - (0, 42) train_loss : 0.0017921136459335685 
+DLL 2023-07-12 19:58:56.202697 - (0, 42) train_items_per_sec : 88229.29075509075 items/s
+DLL 2023-07-12 19:58:56.202754 - (0, 42) train_iter_time : 0.7253826869996374 
+DLL 2023-07-12 19:58:56.203796 - (0, 43) glob_iter/iters_per_epoch : 43/78 
+DLL 2023-07-12 19:58:56.415612 - (0, 43) train_loss : 0.0021591430995613337 
+DLL 2023-07-12 19:58:56.929282 - (0, 43) train_items_per_sec : 88216.18777247946 items/s
+DLL 2023-07-12 19:58:56.929314 - (0, 43) train_iter_time : 0.7254904299998088 
+DLL 2023-07-12 19:58:56.930679 - (0, 44) glob_iter/iters_per_epoch : 44/78 
+DLL 2023-07-12 19:58:57.138654 - (0, 44) train_loss : 0.0013460414484143257 
+DLL 2023-07-12 19:58:57.653441 - (0, 44) train_items_per_sec : 88548.18731686215 items/s
+DLL 2023-07-12 19:58:57.653484 - (0, 44) train_iter_time : 0.722770300999855 
+DLL 2023-07-12 19:58:57.654743 - (0, 45) glob_iter/iters_per_epoch : 45/78 
+DLL 2023-07-12 19:58:57.866211 - (0, 45) train_loss : 0.001953372498974204 
+DLL 2023-07-12 19:58:58.382206 - (0, 45) train_items_per_sec : 87976.2649054696 items/s
+DLL 2023-07-12 19:58:58.382238 - (0, 45) train_iter_time : 0.7274689380001291 
+DLL 2023-07-12 19:58:58.383327 - (0, 46) glob_iter/iters_per_epoch : 46/78 
+DLL 2023-07-12 19:58:58.596171 - (0, 46) train_loss : 0.002875176491215825 
+DLL 2023-07-12 19:58:59.108927 - (0, 46) train_items_per_sec : 88202.11804209843 items/s
+DLL 2023-07-12 19:58:59.108959 - (0, 46) train_iter_time : 0.7256061580001187 
+DLL 2023-07-12 19:58:59.110415 - (0, 47) glob_iter/iters_per_epoch : 47/78 
+DLL 2023-07-12 19:58:59.314603 - (0, 47) train_loss : 0.002048682887107134 
+DLL 2023-07-12 19:58:59.836160 - (0, 47) train_items_per_sec : 88184.62589011276 items/s
+DLL 2023-07-12 19:58:59.836192 - (0, 47) train_iter_time : 0.7257500880000407 
+DLL 2023-07-12 19:58:59.837578 - (0, 48) glob_iter/iters_per_epoch : 48/78 
+DLL 2023-07-12 19:59:00.050427 - (0, 48) train_loss : 0.0011827906128019094 
+DLL 2023-07-12 19:59:00.564527 - (0, 48) train_items_per_sec : 88038.6318358705 items/s
+DLL 2023-07-12 19:59:00.564561 - (0, 48) train_iter_time : 0.7269535959999303 
+DLL 2023-07-12 19:59:00.565996 - (0, 49) glob_iter/iters_per_epoch : 49/78 
+DLL 2023-07-12 19:59:00.778834 - (0, 49) train_loss : 0.001782482722774148 
+DLL 2023-07-12 19:59:01.293638 - (0, 49) train_items_per_sec : 87954.56936499455 items/s
+DLL 2023-07-12 19:59:01.293669 - (0, 49) train_iter_time : 0.7276483810001082 
+DLL 2023-07-12 19:59:01.295205 - (0, 50) glob_iter/iters_per_epoch : 50/78 
+DLL 2023-07-12 19:59:01.499445 - (0, 50) train_loss : 0.002706709783524275 
+DLL 2023-07-12 19:59:02.018549 - (0, 50) train_items_per_sec : 88477.01670208748 items/s
+DLL 2023-07-12 19:59:02.018582 - (0, 50) train_iter_time : 0.7233516950000194 
+DLL 2023-07-12 19:59:02.019690 - (0, 51) glob_iter/iters_per_epoch : 51/78 
+DLL 2023-07-12 19:59:02.231683 - (0, 51) train_loss : 0.0026016111951321363 
+DLL 2023-07-12 19:59:02.745637 - (0, 51) train_items_per_sec : 88159.88512570773 items/s
+DLL 2023-07-12 19:59:02.745670 - (0, 51) train_iter_time : 0.7259537589998217 
+DLL 2023-07-12 19:59:02.747154 - (0, 52) glob_iter/iters_per_epoch : 52/78 
+DLL 2023-07-12 19:59:02.958508 - (0, 52) train_loss : 0.0020805909298360348 
+DLL 2023-07-12 19:59:03.476558 - (0, 52) train_items_per_sec : 87742.3277203467 items/s
+DLL 2023-07-12 19:59:03.476590 - (0, 52) train_iter_time : 0.7294085040002756 
+DLL 2023-07-12 19:59:03.477991 - (0, 53) glob_iter/iters_per_epoch : 53/78 
+DLL 2023-07-12 19:59:03.689719 - (0, 53) train_loss : 0.00204215245321393 
+DLL 2023-07-12 19:59:04.204610 - (0, 53) train_items_per_sec : 88077.90279117506 items/s
+DLL 2023-07-12 19:59:04.204643 - (0, 53) train_iter_time : 0.7266294719997859 
+DLL 2023-07-12 19:59:04.205980 - (0, 54) glob_iter/iters_per_epoch : 54/78 
+DLL 2023-07-12 19:59:04.417640 - (0, 54) train_loss : 0.0015285826520994306 
+DLL 2023-07-12 19:59:04.932507 - (0, 54) train_items_per_sec : 88089.67948901045 items/s
+DLL 2023-07-12 19:59:04.932539 - (0, 54) train_iter_time : 0.7265323289998378 
+DLL 2023-07-12 19:59:04.933941 - (0, 55) glob_iter/iters_per_epoch : 55/78 
+DLL 2023-07-12 19:59:05.142963 - (0, 55) train_loss : 0.0019086562097072601 
+DLL 2023-07-12 19:59:05.658650 - (0, 55) train_items_per_sec : 88310.6161334106 items/s
+DLL 2023-07-12 19:59:05.658684 - (0, 55) train_iter_time : 0.7247146809995684 
+DLL 2023-07-12 19:59:05.659806 - (0, 56) glob_iter/iters_per_epoch : 56/78 
+DLL 2023-07-12 19:59:05.871843 - (0, 56) train_loss : 0.0018373089842498302 
+DLL 2023-07-12 19:59:06.387705 - (0, 56) train_items_per_sec : 87923.55698450479 items/s
+DLL 2023-07-12 19:59:06.387736 - (0, 56) train_iter_time : 0.7279050370002551 
+DLL 2023-07-12 19:59:06.389089 - (0, 57) glob_iter/iters_per_epoch : 57/78 
+DLL 2023-07-12 19:59:06.600784 - (0, 57) train_loss : 0.0017965922597795725 
+DLL 2023-07-12 19:59:07.115719 - (0, 57) train_items_per_sec : 88077.13878381776 items/s
+DLL 2023-07-12 19:59:07.115750 - (0, 57) train_iter_time : 0.7266357750004317 
+DLL 2023-07-12 19:59:07.117092 - (0, 58) glob_iter/iters_per_epoch : 58/78 
+DLL 2023-07-12 19:59:07.324293 - (0, 58) train_loss : 0.002234012121334672 
+DLL 2023-07-12 19:59:07.840715 - (0, 58) train_items_per_sec : 88443.01619782345 items/s
+DLL 2023-07-12 19:59:07.840747 - (0, 58) train_iter_time : 0.7236297760000525 
+DLL 2023-07-12 19:59:07.842174 - (0, 59) glob_iter/iters_per_epoch : 59/78 
+DLL 2023-07-12 19:59:08.053357 - (0, 59) train_loss : 0.003622223623096943 
+DLL 2023-07-12 19:59:08.570190 - (0, 59) train_items_per_sec : 87909.35161275485 items/s
+DLL 2023-07-12 19:59:08.570222 - (0, 59) train_iter_time : 0.7280226600000788 
+DLL 2023-07-12 19:59:08.571510 - (0, 60) glob_iter/iters_per_epoch : 60/78 
+DLL 2023-07-12 19:59:08.784160 - (0, 60) train_loss : 0.0016944969538599253 
+DLL 2023-07-12 19:59:09.298500 - (0, 60) train_items_per_sec : 88033.60851368279 items/s
+DLL 2023-07-12 19:59:09.298532 - (0, 60) train_iter_time : 0.7269950770000833 
+DLL 2023-07-12 19:59:09.299636 - (0, 61) glob_iter/iters_per_epoch : 61/78 
+DLL 2023-07-12 19:59:09.501838 - (0, 61) train_loss : 0.002098578028380871 
+DLL 2023-07-12 19:59:10.024703 - (0, 61) train_items_per_sec : 88267.0723730335 items/s
+DLL 2023-07-12 19:59:10.024743 - (0, 61) train_iter_time : 0.7250721959999282 
+DLL 2023-07-12 19:59:10.026325 - (0, 62) glob_iter/iters_per_epoch : 62/78 
+DLL 2023-07-12 19:59:10.237501 - (0, 62) train_loss : 0.0031142202205955982 
+DLL 2023-07-12 19:59:10.753261 - (0, 62) train_items_per_sec : 88040.6003410488 items/s
+DLL 2023-07-12 19:59:10.753292 - (0, 62) train_iter_time : 0.7269373419999283 
+DLL 2023-07-12 19:59:10.754637 - (0, 63) glob_iter/iters_per_epoch : 63/78 
+DLL 2023-07-12 19:59:10.967648 - (0, 63) train_loss : 0.0015505487099289894 
+DLL 2023-07-12 19:59:11.482534 - (0, 63) train_items_per_sec : 87923.92454963778 items/s
+DLL 2023-07-12 19:59:11.482606 - (0, 63) train_iter_time : 0.727901994000149 
+DLL 2023-07-12 19:59:11.483712 - (0, 64) glob_iter/iters_per_epoch : 64/78 
+DLL 2023-07-12 19:59:11.691740 - (0, 64) train_loss : 0.001944393035955727 
+DLL 2023-07-12 19:59:12.210892 - (0, 64) train_items_per_sec : 88010.52526673098 items/s
+DLL 2023-07-12 19:59:12.210928 - (0, 64) train_iter_time : 0.7271857519999685 
+DLL 2023-07-12 19:59:12.211947 - (0, 65) glob_iter/iters_per_epoch : 65/78 
+DLL 2023-07-12 19:59:12.423920 - (0, 65) train_loss : 0.0022334905806928873 
+DLL 2023-07-12 19:59:12.938841 - (0, 65) train_items_per_sec : 88044.94919171649 items/s
+DLL 2023-07-12 19:59:12.938872 - (0, 65) train_iter_time : 0.7269014359999346 
+DLL 2023-07-12 19:59:12.939901 - (0, 66) glob_iter/iters_per_epoch : 66/78 
+DLL 2023-07-12 19:59:13.151385 - (0, 66) train_loss : 0.0019083620281890035 
+DLL 2023-07-12 19:59:13.668206 - (0, 66) train_items_per_sec : 87874.73740676275 items/s
+DLL 2023-07-12 19:59:13.668237 - (0, 66) train_iter_time : 0.7283094310000706 
+DLL 2023-07-12 19:59:13.669597 - (0, 67) glob_iter/iters_per_epoch : 67/78 
+DLL 2023-07-12 19:59:13.880262 - (0, 67) train_loss : 0.001963644986972213 
+DLL 2023-07-12 19:59:14.396434 - (0, 67) train_items_per_sec : 88052.27027106273 items/s
+DLL 2023-07-12 19:59:14.396465 - (0, 67) train_iter_time : 0.7268409980001707 
+DLL 2023-07-12 19:59:14.397775 - (0, 68) glob_iter/iters_per_epoch : 68/78 
+DLL 2023-07-12 19:59:14.609868 - (0, 68) train_loss : 0.001565529964864254 
+DLL 2023-07-12 19:59:15.126208 - (0, 68) train_items_per_sec : 87859.28484394 items/s
+DLL 2023-07-12 19:59:15.126241 - (0, 68) train_iter_time : 0.7284375250001176 
+DLL 2023-07-12 19:59:15.127074 - (0, 69) glob_iter/iters_per_epoch : 69/78 
+DLL 2023-07-12 19:59:15.336644 - (0, 69) train_loss : 0.0029949576128274202 
+DLL 2023-07-12 19:59:15.855697 - (0, 69) train_items_per_sec : 87836.56547810123 items/s
+DLL 2023-07-12 19:59:15.855745 - (0, 69) train_iter_time : 0.7286259389998122 
+DLL 2023-07-12 19:59:15.856997 - (0, 70) glob_iter/iters_per_epoch : 70/78 
+DLL 2023-07-12 19:59:16.068889 - (0, 70) train_loss : 0.001709540607407689 
+DLL 2023-07-12 19:59:16.587987 - (0, 70) train_items_per_sec : 87551.77567484425 items/s
+DLL 2023-07-12 19:59:16.588020 - (0, 70) train_iter_time : 0.7309960249999676 
+DLL 2023-07-12 19:59:16.589370 - (0, 71) glob_iter/iters_per_epoch : 71/78 
+DLL 2023-07-12 19:59:16.801246 - (0, 71) train_loss : 0.003047792473807931 
+DLL 2023-07-12 19:59:17.315867 - (0, 71) train_items_per_sec : 88093.12290987454 items/s
+DLL 2023-07-12 19:59:17.315907 - (0, 71) train_iter_time : 0.7265039300000353 
+DLL 2023-07-12 19:59:17.317653 - (0, 72) glob_iter/iters_per_epoch : 72/78 
+DLL 2023-07-12 19:59:17.524556 - (0, 72) train_loss : 0.003262902842834592 
+DLL 2023-07-12 19:59:18.044935 - (0, 72) train_items_per_sec : 87997.75336233221 items/s
+DLL 2023-07-12 19:59:18.044972 - (0, 72) train_iter_time : 0.7272912950002137 
+DLL 2023-07-12 19:59:18.046329 - (0, 73) glob_iter/iters_per_epoch : 73/78 
+DLL 2023-07-12 19:59:18.257681 - (0, 73) train_loss : 0.002809758996590972 
+DLL 2023-07-12 19:59:18.773509 - (0, 73) train_items_per_sec : 88010.62015351433 items/s
+DLL 2023-07-12 19:59:18.773546 - (0, 73) train_iter_time : 0.7271849680000742 
+DLL 2023-07-12 19:59:18.774953 - (0, 74) glob_iter/iters_per_epoch : 74/78 
+DLL 2023-07-12 19:59:18.987453 - (0, 74) train_loss : 0.002148660831153393 
+DLL 2023-07-12 19:59:19.504133 - (0, 74) train_items_per_sec : 87769.27849497642 items/s
+DLL 2023-07-12 19:59:19.504195 - (0, 74) train_iter_time : 0.7291845289996672 
+DLL 2023-07-12 19:59:19.505529 - (0, 75) glob_iter/iters_per_epoch : 75/78 
+DLL 2023-07-12 19:59:19.710051 - (0, 75) train_loss : 0.0019846989307552576 
+DLL 2023-07-12 19:59:20.230010 - (0, 75) train_items_per_sec : 88338.71805212922 items/s
+DLL 2023-07-12 19:59:20.230049 - (0, 75) train_iter_time : 0.7244841379997524 
+DLL 2023-07-12 19:59:20.231332 - (0, 76) glob_iter/iters_per_epoch : 76/78 
+DLL 2023-07-12 19:59:20.443440 - (0, 76) train_loss : 0.001315371599048376 
+DLL 2023-07-12 19:59:20.960564 - (0, 76) train_items_per_sec : 87762.92662066247 items/s
+DLL 2023-07-12 19:59:20.960627 - (0, 76) train_iter_time : 0.7292373039999802 
+DLL 2023-07-12 19:59:20.962259 - (0, 77) glob_iter/iters_per_epoch : 77/78 
+DLL 2023-07-12 19:59:21.175468 - (0, 77) train_loss : 0.0012396262027323246 
+DLL 2023-07-12 19:59:21.690802 - (0, 77) train_items_per_sec : 87845.84719594325 items/s
+DLL 2023-07-12 19:59:21.690840 - (0, 77) train_iter_time : 0.7285489529999722 
+DLL 2023-07-12 19:59:21.715615 - (0,) train_items_per_sec : 87535.2311974363 items/s
+DLL 2023-07-12 19:59:21.715679 - (0,) train_loss : 0.0012396262027323246 
+DLL 2023-07-12 19:59:21.715704 - (0,) train_epoch_time : 59.41119023900001 
+DLL 2023-07-12 19:59:22.056094 - (0, 78, 0) val_items_per_sec : 207887.02790045823 items/s
+DLL 2023-07-12 19:59:22.266809 - (0, 78, 1) val_items_per_sec : 306591.0679516606 items/s
+DLL 2023-07-12 19:59:22.476758 - (0, 78, 2) val_items_per_sec : 306037.7762510402 items/s
+DLL 2023-07-12 19:59:22.686517 - (0, 78, 3) val_items_per_sec : 306358.72374853893 items/s
+DLL 2023-07-12 19:59:22.900395 - (0, 78, 4) val_items_per_sec : 300715.04210639175 items/s
+DLL 2023-07-12 19:59:23.115123 - (0, 78, 5) val_items_per_sec : 299670.657265213 items/s
+DLL 2023-07-12 19:59:23.327652 - (0, 78, 6) val_items_per_sec : 302121.2485174097 items/s
+DLL 2023-07-12 19:59:23.539405 - (0, 78, 7) val_items_per_sec : 303598.3899545649 items/s
+DLL 2023-07-12 19:59:23.750532 - (0, 78, 8) val_items_per_sec : 304137.3427973244 items/s
+DLL 2023-07-12 19:59:23.964089 - (0, 78, 9) val_items_per_sec : 300984.3717397505 items/s
+DLL 2023-07-12 19:59:24.178380 - (0, 78, 10) val_items_per_sec : 300012.30988000217 items/s
+DLL 2023-07-12 19:59:24.391262 - (0, 78, 11) val_items_per_sec : 302048.4510636811 items/s
+DLL 2023-07-12 19:59:25.284364 - (0, 78, 12) val_items_per_sec : 35871.14462487776 items/s
+DLL 2023-07-12 19:59:25.296298 - (0,) val_loss : 0.0017842969570595485 None
+DLL 2023-07-12 19:59:25.296355 - (0,) val_items_per_sec : 275079.50413853186 items/s
+Saving model and optimizer state at epoch 0 to ./checkpoint_WaveGlow_0.pt
+DLL 2023-07-12 19:59:27.403179 - (1, 0) glob_iter/iters_per_epoch : 78/78 
+DLL 2023-07-12 19:59:27.618774 - (1, 0) train_loss : 0.002064209198579192 
+DLL 2023-07-12 19:59:28.131653 - (1, 0) train_items_per_sec : 87849.76815539466 items/s
+DLL 2023-07-12 19:59:28.131691 - (1, 0) train_iter_time : 0.7285164360000635 
+DLL 2023-07-12 19:59:28.133879 - (1, 1) glob_iter/iters_per_epoch : 79/78 
+DLL 2023-07-12 19:59:28.339281 - (1, 1) train_loss : 0.00216514035128057 
+DLL 2023-07-12 19:59:28.860636 - (1, 1) train_items_per_sec : 88061.8456690792 items/s
+DLL 2023-07-12 19:59:28.860674 - (1, 1) train_iter_time : 0.7267619650001507 
+DLL 2023-07-12 19:59:28.863434 - (1, 2) glob_iter/iters_per_epoch : 80/78 
+DLL 2023-07-12 19:59:29.074143 - (1, 2) train_loss : 0.002356392564252019 
+DLL 2023-07-12 19:59:29.590720 - (1, 2) train_items_per_sec : 87997.96449708093 items/s
+DLL 2023-07-12 19:59:29.590773 - (1, 2) train_iter_time : 0.7272895500000232 
+DLL 2023-07-12 19:59:29.590921 - (1, 3) glob_iter/iters_per_epoch : 81/78 
+DLL 2023-07-12 19:59:29.803233 - (1, 3) train_loss : 0.0014311156701296568 
+DLL 2023-07-12 19:59:30.321731 - (1, 3) train_items_per_sec : 87573.12782066799 items/s
+DLL 2023-07-12 19:59:30.321772 - (1, 3) train_iter_time : 0.7308177929999147 
+DLL 2023-07-12 19:59:30.323861 - (1, 4) glob_iter/iters_per_epoch : 82/78 
+DLL 2023-07-12 19:59:30.533783 - (1, 4) train_loss : 0.002396577037870884 
+DLL 2023-07-12 19:59:31.049575 - (1, 4) train_items_per_sec : 88188.40896891816 items/s
+DLL 2023-07-12 19:59:31.049622 - (1, 4) train_iter_time : 0.7257189549995928 
+DLL 2023-07-12 19:59:31.051654 - (1, 5) glob_iter/iters_per_epoch : 83/78 
+DLL 2023-07-12 19:59:31.263763 - (1, 5) train_loss : 0.0021904290188103914 
+DLL 2023-07-12 19:59:31.782032 - (1, 5) train_items_per_sec : 87625.15776821795 items/s
+DLL 2023-07-12 19:59:31.782076 - (1, 5) train_iter_time : 0.7303838490001908 
+DLL 2023-07-12 19:59:31.783321 - (1, 6) glob_iter/iters_per_epoch : 84/78 
+DLL 2023-07-12 19:59:31.996097 - (1, 6) train_loss : 0.0016904021613299847 
+DLL 2023-07-12 19:59:32.511041 - (1, 6) train_items_per_sec : 87945.32689671768 items/s
+DLL 2023-07-12 19:59:32.511102 - (1, 6) train_iter_time : 0.7277248519999375 
+DLL 2023-07-12 19:59:32.515975 - (1, 7) glob_iter/iters_per_epoch : 85/78 
+DLL 2023-07-12 19:59:32.723565 - (1, 7) train_loss : 0.001342735136859119 
+DLL 2023-07-12 19:59:33.245075 - (1, 7) train_items_per_sec : 87776.9916464109 items/s
+DLL 2023-07-12 19:59:33.245128 - (1, 7) train_iter_time : 0.7291204540001672 
+DLL 2023-07-12 19:59:33.245295 - (1, 8) glob_iter/iters_per_epoch : 86/78 
+DLL 2023-07-12 19:59:33.459486 - (1, 8) train_loss : 0.0014008822618052363 
+DLL 2023-07-12 19:59:33.975866 - (1, 8) train_items_per_sec : 87602.447674854 items/s
+DLL 2023-07-12 19:59:33.975932 - (1, 8) train_iter_time : 0.7305731939995894 
+DLL 2023-07-12 19:59:33.976081 - (1, 9) glob_iter/iters_per_epoch : 87/78 
+DLL 2023-07-12 19:59:34.189033 - (1, 9) train_loss : 0.0023097482044249773 
+DLL 2023-07-12 19:59:34.708293 - (1, 9) train_items_per_sec : 87405.97275164137 items/s
+DLL 2023-07-12 19:59:34.708341 - (1, 9) train_iter_time : 0.7322154080002292 
+DLL 2023-07-12 19:59:34.708475 - (1, 10) glob_iter/iters_per_epoch : 88/78 
+DLL 2023-07-12 19:59:34.904270 - (1, 10) train_loss : 0.0019005597569048405 
+DLL 2023-07-12 19:59:35.429467 - (1, 10) train_items_per_sec : 88766.08421682558 items/s
+DLL 2023-07-12 19:59:35.429511 - (1, 10) train_iter_time : 0.7209960939999291 
+DLL 2023-07-12 19:59:35.429681 - (1, 11) glob_iter/iters_per_epoch : 89/78 
+DLL 2023-07-12 19:59:35.644912 - (1, 11) train_loss : 0.0019105587853118777 
+DLL 2023-07-12 19:59:36.161991 - (1, 11) train_items_per_sec : 87394.29185878289 items/s
+DLL 2023-07-12 19:59:36.162039 - (1, 11) train_iter_time : 0.7323132739998073 
+DLL 2023-07-12 19:59:36.162172 - (1, 12) glob_iter/iters_per_epoch : 90/78 
+DLL 2023-07-12 19:59:36.374046 - (1, 12) train_loss : 0.0024257067125290632 
+DLL 2023-07-12 19:59:36.895167 - (1, 12) train_items_per_sec : 87312.59257530227 items/s
+DLL 2023-07-12 19:59:36.895207 - (1, 12) train_iter_time : 0.7329985070000475 
+DLL 2023-07-12 19:59:36.895338 - (1, 13) glob_iter/iters_per_epoch : 91/78 
+DLL 2023-07-12 19:59:37.101111 - (1, 13) train_loss : 0.0019575366750359535 
+DLL 2023-07-12 19:59:37.618529 - (1, 13) train_items_per_sec : 88496.09332473557 items/s
+DLL 2023-07-12 19:59:37.618575 - (1, 13) train_iter_time : 0.7231957660001171 
+DLL 2023-07-12 19:59:37.618725 - (1, 14) glob_iter/iters_per_epoch : 92/78 
+DLL 2023-07-12 19:59:37.832441 - (1, 14) train_loss : 0.0024386083241552114 
+DLL 2023-07-12 19:59:38.350567 - (1, 14) train_items_per_sec : 87450.11526028618 items/s
+DLL 2023-07-12 19:59:38.350619 - (1, 14) train_iter_time : 0.7318458049999208 
+DLL 2023-07-12 19:59:38.351441 - (1, 15) glob_iter/iters_per_epoch : 93/78 
+DLL 2023-07-12 19:59:38.565407 - (1, 15) train_loss : 0.0028759988490492105 
+DLL 2023-07-12 19:59:39.082524 - (1, 15) train_items_per_sec : 87540.43990465644 items/s
+DLL 2023-07-12 19:59:39.082568 - (1, 15) train_iter_time : 0.7310906829998203 
+DLL 2023-07-12 19:59:39.082695 - (1, 16) glob_iter/iters_per_epoch : 94/78 
+DLL 2023-07-12 19:59:39.288334 - (1, 16) train_loss : 0.0026141097769141197 
+DLL 2023-07-12 19:59:39.811608 - (1, 16) train_items_per_sec : 87801.64429814632 items/s
+DLL 2023-07-12 19:59:39.811656 - (1, 16) train_iter_time : 0.7289157340001111 
+DLL 2023-07-12 19:59:39.813144 - (1, 17) glob_iter/iters_per_epoch : 95/78 
+DLL 2023-07-12 19:59:40.027315 - (1, 17) train_loss : 0.0015371122863143682 
+DLL 2023-07-12 19:59:40.544020 - (1, 17) train_items_per_sec : 87565.60788178969 items/s
+DLL 2023-07-12 19:59:40.544069 - (1, 17) train_iter_time : 0.7308805540001231 
+DLL 2023-07-12 19:59:40.545594 - (1, 18) glob_iter/iters_per_epoch : 96/78 
+DLL 2023-07-12 19:59:40.760516 - (1, 18) train_loss : 0.002796137472614646 
+DLL 2023-07-12 19:59:41.279797 - (1, 18) train_items_per_sec : 87168.69816723761 items/s
+DLL 2023-07-12 19:59:41.279841 - (1, 18) train_iter_time : 0.7342085099999167 
+DLL 2023-07-12 19:59:41.281213 - (1, 19) glob_iter/iters_per_epoch : 97/78 
+DLL 2023-07-12 19:59:41.486809 - (1, 19) train_loss : 0.0013648545136675239 
+DLL 2023-07-12 19:59:42.007924 - (1, 19) train_items_per_sec : 88067.45850629418 items/s
+DLL 2023-07-12 19:59:42.007967 - (1, 19) train_iter_time : 0.7267156460002298 
+DLL 2023-07-12 19:59:42.009530 - (1, 20) glob_iter/iters_per_epoch : 98/78 
+DLL 2023-07-12 19:59:42.224222 - (1, 20) train_loss : 0.0012032374506816268 
+DLL 2023-07-12 19:59:42.741761 - (1, 20) train_items_per_sec : 87403.38985797239 items/s
+DLL 2023-07-12 19:59:42.741802 - (1, 20) train_iter_time : 0.7322370460001366 
+DLL 2023-07-12 19:59:42.743443 - (1, 21) glob_iter/iters_per_epoch : 99/78 
+DLL 2023-07-12 19:59:42.957582 - (1, 21) train_loss : 0.001737200771458447 
+DLL 2023-07-12 19:59:43.475794 - (1, 21) train_items_per_sec : 87388.90136319525 items/s
+DLL 2023-07-12 19:59:43.475837 - (1, 21) train_iter_time : 0.7323584460000347 
+DLL 2023-07-12 19:59:43.477418 - (1, 22) glob_iter/iters_per_epoch : 100/78 
+DLL 2023-07-12 19:59:43.683515 - (1, 22) train_loss : 0.004173004534095526 
+DLL 2023-07-12 19:59:44.205593 - (1, 22) train_items_per_sec : 87889.9616142512 items/s
+DLL 2023-07-12 19:59:44.205626 - (1, 22) train_iter_time : 0.7281832740000027 
+DLL 2023-07-12 19:59:44.206867 - (1, 23) glob_iter/iters_per_epoch : 101/78 
+DLL 2023-07-12 19:59:44.421285 - (1, 23) train_loss : 0.0020355363376438618 
+DLL 2023-07-12 19:59:44.937997 - (1, 23) train_items_per_sec : 87535.03387992359 items/s
+DLL 2023-07-12 19:59:44.938032 - (1, 23) train_iter_time : 0.7311358339998151 
+DLL 2023-07-12 19:59:44.939479 - (1, 24) glob_iter/iters_per_epoch : 102/78 
+DLL 2023-07-12 19:59:45.152761 - (1, 24) train_loss : 0.002298942534253001 
+DLL 2023-07-12 19:59:45.672302 - (1, 24) train_items_per_sec : 87333.00542613004 items/s
+DLL 2023-07-12 19:59:45.672346 - (1, 24) train_iter_time : 0.7328271789997416 
+DLL 2023-07-12 19:59:45.673793 - (1, 25) glob_iter/iters_per_epoch : 103/78 
+DLL 2023-07-12 19:59:45.883463 - (1, 25) train_loss : 0.0011396801564842463 
+DLL 2023-07-12 19:59:46.401333 - (1, 25) train_items_per_sec : 87966.78041196172 items/s
+DLL 2023-07-12 19:59:46.401382 - (1, 25) train_iter_time : 0.7275473730001067 
+DLL 2023-07-12 19:59:46.403027 - (1, 26) glob_iter/iters_per_epoch : 104/78 
+DLL 2023-07-12 19:59:46.618030 - (1, 26) train_loss : 0.0023600799031555653 
+DLL 2023-07-12 19:59:47.136218 - (1, 26) train_items_per_sec : 87288.8848420619 items/s
+DLL 2023-07-12 19:59:47.136255 - (1, 26) train_iter_time : 0.7331975900001453 
+DLL 2023-07-12 19:59:47.137776 - (1, 27) glob_iter/iters_per_epoch : 105/78 
+DLL 2023-07-12 19:59:47.350316 - (1, 27) train_loss : 0.003204124979674816 
+DLL 2023-07-12 19:59:47.868391 - (1, 27) train_items_per_sec : 87596.79755802745 items/s
+DLL 2023-07-12 19:59:47.868426 - (1, 27) train_iter_time : 0.7306203169996479 
+DLL 2023-07-12 19:59:47.869983 - (1, 28) glob_iter/iters_per_epoch : 106/78 
+DLL 2023-07-12 19:59:48.080459 - (1, 28) train_loss : 0.0019045522203668952 
+DLL 2023-07-12 19:59:48.600749 - (1, 28) train_items_per_sec : 87578.45645891264 items/s
+DLL 2023-07-12 19:59:48.600784 - (1, 28) train_iter_time : 0.7307733269999517 
+DLL 2023-07-12 19:59:48.602205 - (1, 29) glob_iter/iters_per_epoch : 107/78 
+DLL 2023-07-12 19:59:48.817104 - (1, 29) train_loss : 0.0019828309305012226 
+DLL 2023-07-12 19:59:49.335376 - (1, 29) train_items_per_sec : 87291.14832373345 items/s
+DLL 2023-07-12 19:59:49.335409 - (1, 29) train_iter_time : 0.7331785780002065 
+DLL 2023-07-12 19:59:49.337038 - (1, 30) glob_iter/iters_per_epoch : 108/78 
+DLL 2023-07-12 19:59:49.550465 - (1, 30) train_loss : 0.0029301359318196774 
+DLL 2023-07-12 19:59:50.068754 - (1, 30) train_items_per_sec : 87464.91703595659 items/s
+DLL 2023-07-12 19:59:50.068802 - (1, 30) train_iter_time : 0.7317219540000224 
+DLL 2023-07-12 19:59:50.070280 - (1, 31) glob_iter/iters_per_epoch : 109/78 
+DLL 2023-07-12 19:59:50.281030 - (1, 31) train_loss : 0.003451918251812458 
+DLL 2023-07-12 19:59:50.801219 - (1, 31) train_items_per_sec : 87557.86858576079 items/s
+DLL 2023-07-12 19:59:50.801256 - (1, 31) train_iter_time : 0.730945156999951 
+DLL 2023-07-12 19:59:50.802794 - (1, 32) glob_iter/iters_per_epoch : 110/78 
+DLL 2023-07-12 19:59:51.018482 - (1, 32) train_loss : 0.0022742431610822678 
+DLL 2023-07-12 19:59:51.535825 - (1, 32) train_items_per_sec : 87308.04242339723 items/s
+DLL 2023-07-12 19:59:51.535858 - (1, 32) train_iter_time : 0.733036708000327 
+DLL 2023-07-12 19:59:51.537373 - (1, 33) glob_iter/iters_per_epoch : 111/78 
+DLL 2023-07-12 19:59:51.751008 - (1, 33) train_loss : 0.002770790131762624 
+DLL 2023-07-12 19:59:52.269220 - (1, 33) train_items_per_sec : 87449.28933562555 items/s
+DLL 2023-07-12 19:59:52.269265 - (1, 33) train_iter_time : 0.731852717000038 
+DLL 2023-07-12 19:59:52.270586 - (1, 34) glob_iter/iters_per_epoch : 112/78 
+DLL 2023-07-12 19:59:52.479634 - (1, 34) train_loss : 0.002675203839316964 
+DLL 2023-07-12 19:59:52.999875 - (1, 34) train_items_per_sec : 87756.03501954176 items/s
+DLL 2023-07-12 19:59:52.999911 - (1, 34) train_iter_time : 0.7292945720000716 
+DLL 2023-07-12 19:59:53.001245 - (1, 35) glob_iter/iters_per_epoch : 113/78 
+DLL 2023-07-12 19:59:53.216286 - (1, 35) train_loss : 0.0017569221090525389 
+DLL 2023-07-12 19:59:53.734696 - (1, 35) train_items_per_sec : 87257.93114361815 items/s
+DLL 2023-07-12 19:59:53.734731 - (1, 35) train_iter_time : 0.7334576830003243 
+DLL 2023-07-12 19:59:53.736336 - (1, 36) glob_iter/iters_per_epoch : 114/78 
+DLL 2023-07-12 19:59:53.949624 - (1, 36) train_loss : 0.0028066346421837807 
+DLL 2023-07-12 19:59:54.467206 - (1, 36) train_items_per_sec : 87565.98983253191 items/s
+DLL 2023-07-12 19:59:54.467242 - (1, 36) train_iter_time : 0.7308773660001862 
+DLL 2023-07-12 19:59:54.468156 - (1, 37) glob_iter/iters_per_epoch : 115/78 
+DLL 2023-07-12 19:59:54.679114 - (1, 37) train_loss : 0.0023165319580584764 
+DLL 2023-07-12 19:59:55.199602 - (1, 37) train_items_per_sec : 87497.002133963 items/s
+DLL 2023-07-12 19:59:55.199636 - (1, 37) train_iter_time : 0.7314536320000116 
+DLL 2023-07-12 19:59:55.201204 - (1, 38) glob_iter/iters_per_epoch : 116/78 
+DLL 2023-07-12 19:59:55.416099 - (1, 38) train_loss : 0.002335014520213008 
+DLL 2023-07-12 19:59:55.934930 - (1, 38) train_items_per_sec : 87225.37808637708 items/s
+DLL 2023-07-12 19:59:55.934966 - (1, 38) train_iter_time : 0.7337314139999762 
+DLL 2023-07-12 19:59:55.935742 - (1, 39) glob_iter/iters_per_epoch : 117/78 
+DLL 2023-07-12 19:59:56.149864 - (1, 39) train_loss : 0.001951219281181693 
+DLL 2023-07-12 19:59:56.668437 - (1, 39) train_items_per_sec : 87348.13833507117 items/s
+DLL 2023-07-12 19:59:56.668478 - (1, 39) train_iter_time : 0.7327002179999909 
+DLL 2023-07-12 19:59:56.669885 - (1, 40) glob_iter/iters_per_epoch : 118/78 
+DLL 2023-07-12 19:59:56.879990 - (1, 40) train_loss : 0.0027336659841239452 
+DLL 2023-07-12 19:59:57.399676 - (1, 40) train_items_per_sec : 87695.6539539476 items/s
+DLL 2023-07-12 19:59:57.399711 - (1, 40) train_iter_time : 0.7297967130002689 
+DLL 2023-07-12 19:59:57.401056 - (1, 41) glob_iter/iters_per_epoch : 119/78 
+DLL 2023-07-12 19:59:57.616171 - (1, 41) train_loss : 0.0022355157416313887 
+DLL 2023-07-12 19:59:58.135359 - (1, 41) train_items_per_sec : 87156.86303867758 items/s
+DLL 2023-07-12 19:59:58.135402 - (1, 41) train_iter_time : 0.734308209000119 
+DLL 2023-07-12 19:59:58.136270 - (1, 42) glob_iter/iters_per_epoch : 120/78 
+DLL 2023-07-12 19:59:58.351639 - (1, 42) train_loss : 0.0018937707645818591 
+DLL 2023-07-12 19:59:58.867901 - (1, 42) train_items_per_sec : 87475.22797215288 items/s
+DLL 2023-07-12 19:59:58.867945 - (1, 42) train_iter_time : 0.7316357040003822 
+DLL 2023-07-12 19:59:58.869413 - (1, 43) glob_iter/iters_per_epoch : 121/78 
+DLL 2023-07-12 19:59:59.078090 - (1, 43) train_loss : 0.0016652527265250683 
+DLL 2023-07-12 19:59:59.603110 - (1, 43) train_items_per_sec : 87228.9166682711 items/s
+DLL 2023-07-12 19:59:59.603161 - (1, 43) train_iter_time : 0.7337016490000678 
+DLL 2023-07-12 19:59:59.604349 - (1, 44) glob_iter/iters_per_epoch : 122/78 
+DLL 2023-07-12 19:59:59.818512 - (1, 44) train_loss : 0.0022070242557674646 
+DLL 2023-07-12 20:00:00.335750 - (1, 44) train_items_per_sec : 87502.65529542007 items/s
+DLL 2023-07-12 20:00:00.335803 - (1, 44) train_iter_time : 0.7314063759999954 
+DLL 2023-07-12 20:00:00.337346 - (1, 45) glob_iter/iters_per_epoch : 123/78 
+DLL 2023-07-12 20:00:00.551770 - (1, 45) train_loss : 0.0025057366583496332 
+DLL 2023-07-12 20:00:01.072410 - (1, 45) train_items_per_sec : 87066.66757996748 items/s
+DLL 2023-07-12 20:00:01.072454 - (1, 45) train_iter_time : 0.7350689049999346 
+DLL 2023-07-12 20:00:01.073992 - (1, 46) glob_iter/iters_per_epoch : 124/78 
+DLL 2023-07-12 20:00:01.280724 - (1, 46) train_loss : 0.0026979194954037666 
+DLL 2023-07-12 20:00:01.803266 - (1, 46) train_items_per_sec : 87757.77887693427 items/s
+DLL 2023-07-12 20:00:01.803312 - (1, 46) train_iter_time : 0.7292800800000805 
+DLL 2023-07-12 20:00:01.804134 - (1, 47) glob_iter/iters_per_epoch : 125/78 
+DLL 2023-07-12 20:00:02.017631 - (1, 47) train_loss : 0.0011865223059430718 
+DLL 2023-07-12 20:00:02.538269 - (1, 47) train_items_per_sec : 87176.64264336797 items/s
+DLL 2023-07-12 20:00:02.538309 - (1, 47) train_iter_time : 0.7341416009999193 
+DLL 2023-07-12 20:00:02.539653 - (1, 48) glob_iter/iters_per_epoch : 126/78 
+DLL 2023-07-12 20:00:02.756347 - (1, 48) train_loss : 0.0017293412238359451 
+DLL 2023-07-12 20:00:03.272549 - (1, 48) train_items_per_sec : 87324.31371411715 items/s
+DLL 2023-07-12 20:00:03.272593 - (1, 48) train_iter_time : 0.7329001200000675 
+DLL 2023-07-12 20:00:03.273947 - (1, 49) glob_iter/iters_per_epoch : 127/78 
+DLL 2023-07-12 20:00:03.481388 - (1, 49) train_loss : 0.003093091072514653 
+DLL 2023-07-12 20:00:04.005639 - (1, 49) train_items_per_sec : 87467.80026685314 items/s
+DLL 2023-07-12 20:00:04.005683 - (1, 49) train_iter_time : 0.7316978339999878 
+DLL 2023-07-12 20:00:04.007157 - (1, 50) glob_iter/iters_per_epoch : 128/78 
+DLL 2023-07-12 20:00:04.221433 - (1, 50) train_loss : 0.0019002185435965657 
+DLL 2023-07-12 20:00:04.739571 - (1, 50) train_items_per_sec : 87381.56621665799 items/s
+DLL 2023-07-12 20:00:04.739614 - (1, 50) train_iter_time : 0.7324199229997248 
+DLL 2023-07-12 20:00:04.741243 - (1, 51) glob_iter/iters_per_epoch : 129/78 
+DLL 2023-07-12 20:00:04.956680 - (1, 51) train_loss : 0.0026974112261086702 
+DLL 2023-07-12 20:00:05.477788 - (1, 51) train_items_per_sec : 86891.41468685954 items/s
+DLL 2023-07-12 20:00:05.477829 - (1, 51) train_iter_time : 0.7365514789998997 
+DLL 2023-07-12 20:00:05.479218 - (1, 52) glob_iter/iters_per_epoch : 130/78 
+DLL 2023-07-12 20:00:05.684635 - (1, 52) train_loss : 0.0022784629836678505 
+DLL 2023-07-12 20:00:06.207921 - (1, 52) train_items_per_sec : 87826.50134839758 items/s
+DLL 2023-07-12 20:00:06.207962 - (1, 52) train_iter_time : 0.7287094330004038 
+DLL 2023-07-12 20:00:06.209523 - (1, 53) glob_iter/iters_per_epoch : 131/78 
+DLL 2023-07-12 20:00:06.423804 - (1, 53) train_loss : 0.0024080772418528795 
+DLL 2023-07-12 20:00:06.941821 - (1, 53) train_items_per_sec : 87395.44839834841 items/s
+DLL 2023-07-12 20:00:06.941864 - (1, 53) train_iter_time : 0.7323035829999753 
+DLL 2023-07-12 20:00:06.943441 - (1, 54) glob_iter/iters_per_epoch : 132/78 
+DLL 2023-07-12 20:00:07.158919 - (1, 54) train_loss : 0.0013037491589784622 
+DLL 2023-07-12 20:00:07.678272 - (1, 54) train_items_per_sec : 87094.32093004897 items/s
+DLL 2023-07-12 20:00:07.678320 - (1, 54) train_iter_time : 0.7348355129997799 
+DLL 2023-07-12 20:00:07.679078 - (1, 55) glob_iter/iters_per_epoch : 133/78 
+DLL 2023-07-12 20:00:07.886427 - (1, 55) train_loss : 0.001902301562950015 
+DLL 2023-07-12 20:00:08.409581 - (1, 55) train_items_per_sec : 87610.08068033768 items/s
+DLL 2023-07-12 20:00:08.409621 - (1, 55) train_iter_time : 0.7305095430001529 
+DLL 2023-07-12 20:00:08.411308 - (1, 56) glob_iter/iters_per_epoch : 134/78 
+DLL 2023-07-12 20:00:08.625598 - (1, 56) train_loss : 0.0018380836118012667 
+DLL 2023-07-12 20:00:09.144808 - (1, 56) train_items_per_sec : 87252.23925714617 items/s
+DLL 2023-07-12 20:00:09.144845 - (1, 56) train_iter_time : 0.7335055300000022 
+DLL 2023-07-12 20:00:09.146341 - (1, 57) glob_iter/iters_per_epoch : 135/78 
+DLL 2023-07-12 20:00:09.362138 - (1, 57) train_loss : 0.001049871789291501 
+DLL 2023-07-12 20:00:09.880534 - (1, 57) train_items_per_sec : 87170.07076532119 items/s
+DLL 2023-07-12 20:00:09.880581 - (1, 57) train_iter_time : 0.7341969489998519 
+DLL 2023-07-12 20:00:09.882054 - (1, 58) glob_iter/iters_per_epoch : 136/78 
+DLL 2023-07-12 20:00:10.088621 - (1, 58) train_loss : 0.00184523849748075 
+DLL 2023-07-12 20:00:10.615001 - (1, 58) train_items_per_sec : 87317.81033139459 items/s
+DLL 2023-07-12 20:00:10.615042 - (1, 58) train_iter_time : 0.7329547059998731 
+DLL 2023-07-12 20:00:10.616121 - (1, 59) glob_iter/iters_per_epoch : 137/78 
+DLL 2023-07-12 20:00:10.830904 - (1, 59) train_loss : 0.0026471554301679134 
+DLL 2023-07-12 20:00:11.348497 - (1, 59) train_items_per_sec : 87386.10255830179 items/s
+DLL 2023-07-12 20:00:11.348538 - (1, 59) train_iter_time : 0.7323819019998155 
+DLL 2023-07-12 20:00:11.349885 - (1, 60) glob_iter/iters_per_epoch : 138/78 
+DLL 2023-07-12 20:00:11.566109 - (1, 60) train_loss : 0.0025581035297363997 
+DLL 2023-07-12 20:00:12.086772 - (1, 60) train_items_per_sec : 86851.19047974721 items/s
+DLL 2023-07-12 20:00:12.086816 - (1, 60) train_iter_time : 0.736892605000321 
+DLL 2023-07-12 20:00:12.087980 - (1, 61) glob_iter/iters_per_epoch : 139/78 
+DLL 2023-07-12 20:00:12.295895 - (1, 61) train_loss : 0.0022971872240304947 
+DLL 2023-07-12 20:00:12.817508 - (1, 61) train_items_per_sec : 87727.27197651172 items/s
+DLL 2023-07-12 20:00:12.817552 - (1, 61) train_iter_time : 0.7295336849997511 
+DLL 2023-07-12 20:00:12.819166 - (1, 62) glob_iter/iters_per_epoch : 140/78 
+DLL 2023-07-12 20:00:13.033625 - (1, 62) train_loss : 0.0024224743247032166 
+DLL 2023-07-12 20:00:13.553253 - (1, 62) train_items_per_sec : 87182.77489917476 items/s
+DLL 2023-07-12 20:00:13.553304 - (1, 62) train_iter_time : 0.734089963000315 
+DLL 2023-07-12 20:00:13.554017 - (1, 63) glob_iter/iters_per_epoch : 141/78 
+DLL 2023-07-12 20:00:13.769839 - (1, 63) train_loss : 0.002813852857798338 
+DLL 2023-07-12 20:00:14.288366 - (1, 63) train_items_per_sec : 87151.29244443533 items/s
+DLL 2023-07-12 20:00:14.288408 - (1, 63) train_iter_time : 0.734355145000336 
+DLL 2023-07-12 20:00:14.289761 - (1, 64) glob_iter/iters_per_epoch : 142/78 
+DLL 2023-07-12 20:00:14.497698 - (1, 64) train_loss : 0.0017186716431751847 
+DLL 2023-07-12 20:00:15.021046 - (1, 64) train_items_per_sec : 87516.49819319988 items/s
+DLL 2023-07-12 20:00:15.021094 - (1, 64) train_iter_time : 0.7312906859997383 
+DLL 2023-07-12 20:00:15.022657 - (1, 65) glob_iter/iters_per_epoch : 143/78 
+DLL 2023-07-12 20:00:15.237596 - (1, 65) train_loss : 0.002270511817187071 
+DLL 2023-07-12 20:00:15.756140 - (1, 65) train_items_per_sec : 87254.2154580178 items/s
+DLL 2023-07-12 20:00:15.756182 - (1, 65) train_iter_time : 0.7334889170001588 
+DLL 2023-07-12 20:00:15.757649 - (1, 66) glob_iter/iters_per_epoch : 144/78 
+DLL 2023-07-12 20:00:15.973030 - (1, 66) train_loss : 0.0019164145924150944 
+DLL 2023-07-12 20:00:16.494021 - (1, 66) train_items_per_sec : 86911.90180644112 items/s
+DLL 2023-07-12 20:00:16.494067 - (1, 66) train_iter_time : 0.7363778569997521 
+DLL 2023-07-12 20:00:16.495545 - (1, 67) glob_iter/iters_per_epoch : 145/78 
+DLL 2023-07-12 20:00:16.704089 - (1, 67) train_loss : 0.0023430853616446257 
+DLL 2023-07-12 20:00:17.226181 - (1, 67) train_items_per_sec : 87593.76375214933 items/s
+DLL 2023-07-12 20:00:17.226219 - (1, 67) train_iter_time : 0.7306456219998836 
+DLL 2023-07-12 20:00:17.227398 - (1, 68) glob_iter/iters_per_epoch : 146/78 
+DLL 2023-07-12 20:00:17.441520 - (1, 68) train_loss : 0.0009583414066582918 
+DLL 2023-07-12 20:00:17.962074 - (1, 68) train_items_per_sec : 87112.49927300694 items/s
+DLL 2023-07-12 20:00:17.962113 - (1, 68) train_iter_time : 0.7346821699998145 
+DLL 2023-07-12 20:00:17.963288 - (1, 69) glob_iter/iters_per_epoch : 147/78 
+DLL 2023-07-12 20:00:18.178443 - (1, 69) train_loss : 0.0019002009648829699 
+DLL 2023-07-12 20:00:18.699269 - (1, 69) train_items_per_sec : 86958.1275107597 items/s
+DLL 2023-07-12 20:00:18.699305 - (1, 69) train_iter_time : 0.7359864089999064 
+DLL 2023-07-12 20:00:18.700552 - (1, 70) glob_iter/iters_per_epoch : 148/78 
+DLL 2023-07-12 20:00:18.909571 - (1, 70) train_loss : 0.0023570701014250517 
+DLL 2023-07-12 20:00:19.431530 - (1, 70) train_items_per_sec : 87553.26301046762 items/s
+DLL 2023-07-12 20:00:19.431571 - (1, 70) train_iter_time : 0.730983606999871 
+DLL 2023-07-12 20:00:19.432257 - (1, 71) glob_iter/iters_per_epoch : 149/78 
+DLL 2023-07-12 20:00:19.647261 - (1, 71) train_loss : 0.003519991645589471 
+DLL 2023-07-12 20:00:20.166954 - (1, 71) train_items_per_sec : 87109.7853703336 items/s
+DLL 2023-07-12 20:00:20.166988 - (1, 71) train_iter_time : 0.7347050590001345 
+DLL 2023-07-12 20:00:20.168241 - (1, 72) glob_iter/iters_per_epoch : 150/78 
+DLL 2023-07-12 20:00:20.383260 - (1, 72) train_loss : 0.0033279519993811846 
+DLL 2023-07-12 20:00:20.903383 - (1, 72) train_items_per_sec : 87057.43798785203 items/s
+DLL 2023-07-12 20:00:20.903419 - (1, 72) train_iter_time : 0.7351468350002506 
+DLL 2023-07-12 20:00:20.904196 - (1, 73) glob_iter/iters_per_epoch : 151/78 
+DLL 2023-07-12 20:00:21.115272 - (1, 73) train_loss : 0.001951622311025858 
+DLL 2023-07-12 20:00:21.635969 - (1, 73) train_items_per_sec : 87457.9699714692 items/s
+DLL 2023-07-12 20:00:21.636007 - (1, 73) train_iter_time : 0.7317800770001668 
+DLL 2023-07-12 20:00:21.637287 - (1, 74) glob_iter/iters_per_epoch : 152/78 
+DLL 2023-07-12 20:00:21.851864 - (1, 74) train_loss : 0.0035820852499455214 
+DLL 2023-07-12 20:00:22.371513 - (1, 74) train_items_per_sec : 87166.07074968712 items/s
+DLL 2023-07-12 20:00:22.371550 - (1, 74) train_iter_time : 0.7342306410000674 
+DLL 2023-07-12 20:00:22.372921 - (1, 75) glob_iter/iters_per_epoch : 153/78 
+DLL 2023-07-12 20:00:22.587960 - (1, 75) train_loss : 0.002288535237312317 
+DLL 2023-07-12 20:00:23.109398 - (1, 75) train_items_per_sec : 86899.40486191826 items/s
+DLL 2023-07-12 20:00:23.109432 - (1, 75) train_iter_time : 0.7364837550003358 
+DLL 2023-07-12 20:00:23.110782 - (1, 76) glob_iter/iters_per_epoch : 154/78 
+DLL 2023-07-12 20:00:23.322732 - (1, 76) train_loss : 0.0015967515064403415 
+DLL 2023-07-12 20:00:23.841915 - (1, 76) train_items_per_sec : 87534.85513126465 items/s
+DLL 2023-07-12 20:00:23.841955 - (1, 76) train_iter_time : 0.7311373269999422 
+DLL 2023-07-12 20:00:23.843156 - (1, 77) glob_iter/iters_per_epoch : 155/78 
+DLL 2023-07-12 20:00:24.056392 - (1, 77) train_loss : 0.0012947495561093092 
+DLL 2023-07-12 20:00:24.577128 - (1, 77) train_items_per_sec : 87196.1619083045 items/s
+DLL 2023-07-12 20:00:24.577164 - (1, 77) train_iter_time : 0.7339772600003016 
+DLL 2023-07-12 20:00:24.595949 - (1,) train_items_per_sec : 87470.58396767074 items/s
+DLL 2023-07-12 20:00:24.596032 - (1,) train_loss : 0.0012947495561093092 
+DLL 2023-07-12 20:00:24.596045 - (1,) train_epoch_time : 57.34352359600007 
+DLL 2023-07-12 20:00:24.826535 - (1, 156, 0) val_items_per_sec : 319811.418399967 items/s
+DLL 2023-07-12 20:00:25.037629 - (1, 156, 1) val_items_per_sec : 305004.591105489 items/s
+DLL 2023-07-12 20:00:25.252755 - (1, 156, 2) val_items_per_sec : 298762.6302199447 items/s
+DLL 2023-07-12 20:00:25.465707 - (1, 156, 3) val_items_per_sec : 301670.94262486487 items/s
+DLL 2023-07-12 20:00:25.682319 - (1, 156, 4) val_items_per_sec : 296532.92957392085 items/s
+DLL 2023-07-12 20:00:25.896477 - (1, 156, 5) val_items_per_sec : 300113.54608470324 items/s
+DLL 2023-07-12 20:00:26.109117 - (1, 156, 6) val_items_per_sec : 302112.77423316485 items/s
+DLL 2023-07-12 20:00:26.323280 - (1, 156, 7) val_items_per_sec : 300394.66273178253 items/s
+DLL 2023-07-12 20:00:26.537050 - (1, 156, 8) val_items_per_sec : 300283.5685654944 items/s
+DLL 2023-07-12 20:00:26.752564 - (1, 156, 9) val_items_per_sec : 298627.527945407 items/s
+DLL 2023-07-12 20:00:26.967077 - (1, 156, 10) val_items_per_sec : 300041.12485512294 items/s
+DLL 2023-07-12 20:00:27.180090 - (1, 156, 11) val_items_per_sec : 301637.5942698215 items/s
+DLL 2023-07-12 20:00:27.291595 - (1, 156, 12) val_items_per_sec : 289465.63368571387 items/s
+DLL 2023-07-12 20:00:27.303957 - (1,) val_loss : 0.001913126764030984 None
+DLL 2023-07-12 20:00:27.303988 - (1,) val_items_per_sec : 301112.22648426134 items/s
+DLL 2023-07-12 20:00:27.304710 - () run_time : 127.90688520899994 s
+DLL 2023-07-12 20:00:27.304730 - () val_loss : 0.001913126764030984 None
+DLL 2023-07-12 20:00:27.304740 - () train_loss : 0.0012947495561093092 
+DLL 2023-07-12 20:00:27.304749 - () train_items_per_sec : 87470.58396767074 items/s
+DLL 2023-07-12 20:00:27.304758 - () val_items_per_sec : 301112.22648426134 items/s
+DONE!
diff --git a/pytorch/results/4090laptop_v1/PyTorch_waveglow_FP16/benchmark.para b/pytorch/results/4090laptop_v1/PyTorch_waveglow_FP16/benchmark.para
new file mode 100755
index 00000000..6ea5ca92
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_waveglow_FP16/benchmark.para
@@ -0,0 +1,2 @@
+GLOBAL_BATCH 8
+GPU 1
diff --git a/pytorch/results/4090laptop_v1/PyTorch_waveglow_FP32/12-07-2023_20-13-13.txt b/pytorch/results/4090laptop_v1/PyTorch_waveglow_FP32/12-07-2023_20-13-13.txt
new file mode 100755
index 00000000..00ab3473
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_waveglow_FP32/12-07-2023_20-13-13.txt
@@ -0,0 +1,710 @@
+OMP: Info #276: omp_set_nested routine deprecated, please use omp_set_max_active_levels instead.
+DLL 2023-07-12 20:13:19.963460 - PARAMETER output : ./ 
+DLL 2023-07-12 20:13:19.963492 - PARAMETER dataset_path : /data/tacotron2/LJSpeech-1.1 
+DLL 2023-07-12 20:13:19.963504 - PARAMETER model_name : WaveGlow 
+DLL 2023-07-12 20:13:19.963512 - PARAMETER log_file : nvlog.json 
+DLL 2023-07-12 20:13:19.963519 - PARAMETER anneal_steps : None 
+DLL 2023-07-12 20:13:19.963526 - PARAMETER anneal_factor : 0.1 
+DLL 2023-07-12 20:13:19.963534 - PARAMETER config_file : None 
+DLL 2023-07-12 20:13:19.963541 - PARAMETER seed : None 
+DLL 2023-07-12 20:13:19.963547 - PARAMETER epochs : 2 
+DLL 2023-07-12 20:13:19.963554 - PARAMETER epochs_per_checkpoint : 50 
+DLL 2023-07-12 20:13:19.963561 - PARAMETER checkpoint_path :  
+DLL 2023-07-12 20:13:19.963567 - PARAMETER resume_from_last : False 
+DLL 2023-07-12 20:13:19.963574 - PARAMETER dynamic_loss_scaling : True 
+DLL 2023-07-12 20:13:19.963581 - PARAMETER amp : False 
+DLL 2023-07-12 20:13:19.963587 - PARAMETER cudnn_enabled : True 
+DLL 2023-07-12 20:13:19.963594 - PARAMETER cudnn_benchmark : True 
+DLL 2023-07-12 20:13:19.963599 - PARAMETER disable_uniform_initialize_bn_weight : False 
+DLL 2023-07-12 20:13:19.963606 - PARAMETER use_saved_learning_rate : False 
+DLL 2023-07-12 20:13:19.963612 - PARAMETER learning_rate : 0.0 
+DLL 2023-07-12 20:13:19.963619 - PARAMETER weight_decay : 0.0 
+DLL 2023-07-12 20:13:19.963625 - PARAMETER grad_clip_thresh : 65504.0 
+DLL 2023-07-12 20:13:19.963631 - PARAMETER batch_size : 8 
+DLL 2023-07-12 20:13:19.963638 - PARAMETER grad_clip : 5.0 
+DLL 2023-07-12 20:13:19.963644 - PARAMETER load_mel_from_disk : False 
+DLL 2023-07-12 20:13:19.963650 - PARAMETER training_files : filelists/ljs_audio_text_train_subset_625_filelist.txt 
+DLL 2023-07-12 20:13:19.963656 - PARAMETER validation_files : filelists/ljs_audio_text_val_filelist.txt 
+DLL 2023-07-12 20:13:19.963662 - PARAMETER text_cleaners : ['english_cleaners'] 
+DLL 2023-07-12 20:13:19.963670 - PARAMETER max_wav_value : 32768.0 
+DLL 2023-07-12 20:13:19.963676 - PARAMETER sampling_rate : 22050 
+DLL 2023-07-12 20:13:19.963682 - PARAMETER filter_length : 1024 
+DLL 2023-07-12 20:13:19.963688 - PARAMETER hop_length : 256 
+DLL 2023-07-12 20:13:19.963694 - PARAMETER win_length : 1024 
+DLL 2023-07-12 20:13:19.963700 - PARAMETER mel_fmin : 0.0 
+DLL 2023-07-12 20:13:19.963706 - PARAMETER mel_fmax : 8000.0 
+DLL 2023-07-12 20:13:19.963712 - PARAMETER rank : 0 
+DLL 2023-07-12 20:13:19.963718 - PARAMETER world_size : 1 
+DLL 2023-07-12 20:13:19.963724 - PARAMETER dist_url : tcp://localhost:23456 
+DLL 2023-07-12 20:13:19.963729 - PARAMETER group_name : group_name 
+DLL 2023-07-12 20:13:19.963736 - PARAMETER dist_backend : nccl 
+DLL 2023-07-12 20:13:19.963742 - PARAMETER bench_class :  
+DLL 2023-07-12 20:13:19.963747 - PARAMETER model_name : Tacotron2_PyT 
+DLL 2023-07-12 20:13:23.326354 - (0, 0) glob_iter/iters_per_epoch : 0/78 
+DLL 2023-07-12 20:13:25.034070 - (0, 0) train_loss : 0.003365968121215701 
+DLL 2023-07-12 20:13:26.226114 - (0, 0) train_items_per_sec : 22070.446573585315 items/s
+DLL 2023-07-12 20:13:26.226159 - (0, 0) train_iter_time : 2.8998053929999514 
+DLL 2023-07-12 20:13:26.226300 - (0, 1) glob_iter/iters_per_epoch : 1/78 
+DLL 2023-07-12 20:13:26.435654 - (0, 1) train_loss : 0.0015499831642955542 
+DLL 2023-07-12 20:13:26.947012 - (0, 1) train_items_per_sec : 88800.79627895488 items/s
+DLL 2023-07-12 20:13:26.947067 - (0, 1) train_iter_time : 0.7207142580000436 
+DLL 2023-07-12 20:13:26.947187 - (0, 2) glob_iter/iters_per_epoch : 2/78 
+DLL 2023-07-12 20:13:27.137340 - (0, 2) train_loss : 0.002983580809086561 
+DLL 2023-07-12 20:13:27.659788 - (0, 2) train_items_per_sec : 89811.17481153153 items/s
+DLL 2023-07-12 20:13:27.659822 - (0, 2) train_iter_time : 0.7126062000002094 
+DLL 2023-07-12 20:13:27.659935 - (0, 3) glob_iter/iters_per_epoch : 3/78 
+DLL 2023-07-12 20:13:27.871279 - (0, 3) train_loss : 0.003592157270759344 
+DLL 2023-07-12 20:13:28.383215 - (0, 3) train_items_per_sec : 88485.25479639154 items/s
+DLL 2023-07-12 20:13:28.383247 - (0, 3) train_iter_time : 0.7232843500000854 
+DLL 2023-07-12 20:13:28.383354 - (0, 4) glob_iter/iters_per_epoch : 4/78 
+DLL 2023-07-12 20:13:28.588622 - (0, 4) train_loss : 0.0027912601362913847 
+DLL 2023-07-12 20:13:29.105049 - (0, 4) train_items_per_sec : 88679.65871789108 items/s
+DLL 2023-07-12 20:13:29.105080 - (0, 4) train_iter_time : 0.7216987630004041 
+DLL 2023-07-12 20:13:29.111042 - (0, 5) glob_iter/iters_per_epoch : 5/78 
+DLL 2023-07-12 20:13:29.322105 - (0, 5) train_loss : 0.0013539085630327463 
+DLL 2023-07-12 20:13:29.833110 - (0, 5) train_items_per_sec : 88633.75404275506 items/s
+DLL 2023-07-12 20:13:29.833142 - (0, 5) train_iter_time : 0.7220725409997613 
+DLL 2023-07-12 20:13:29.833253 - (0, 6) glob_iter/iters_per_epoch : 6/78 
+DLL 2023-07-12 20:13:30.043708 - (0, 6) train_loss : 0.0019192862091585994 
+DLL 2023-07-12 20:13:30.557705 - (0, 6) train_items_per_sec : 88342.29535323613 items/s
+DLL 2023-07-12 20:13:30.557738 - (0, 6) train_iter_time : 0.7244548009998653 
+DLL 2023-07-12 20:13:30.557856 - (0, 7) glob_iter/iters_per_epoch : 7/78 
+DLL 2023-07-12 20:13:30.762211 - (0, 7) train_loss : 0.0017619662685319781 
+DLL 2023-07-12 20:13:31.277395 - (0, 7) train_items_per_sec : 88945.30207610442 items/s
+DLL 2023-07-12 20:13:31.277427 - (0, 7) train_iter_time : 0.7195433430001685 
+DLL 2023-07-12 20:13:31.277537 - (0, 8) glob_iter/iters_per_epoch : 8/78 
+DLL 2023-07-12 20:13:31.488738 - (0, 8) train_loss : 0.0025939117185771465 
+DLL 2023-07-12 20:13:32.001376 - (0, 8) train_items_per_sec : 88417.0122068099 items/s
+DLL 2023-07-12 20:13:32.001409 - (0, 8) train_iter_time : 0.7238425999998981 
+DLL 2023-07-12 20:13:32.001540 - (0, 9) glob_iter/iters_per_epoch : 9/78 
+DLL 2023-07-12 20:13:32.213982 - (0, 9) train_loss : 0.0015035072574391961 
+DLL 2023-07-12 20:13:32.728257 - (0, 9) train_items_per_sec : 88066.92686841165 items/s
+DLL 2023-07-12 20:13:32.728289 - (0, 9) train_iter_time : 0.726720032999765 
+DLL 2023-07-12 20:13:32.728397 - (0, 10) glob_iter/iters_per_epoch : 10/78 
+DLL 2023-07-12 20:13:32.937865 - (0, 10) train_loss : 0.004458735231310129 
+DLL 2023-07-12 20:13:33.452528 - (0, 10) train_items_per_sec : 88381.28680309295 items/s
+DLL 2023-07-12 20:13:33.452561 - (0, 10) train_iter_time : 0.7241351910001868 
+DLL 2023-07-12 20:13:33.452706 - (0, 11) glob_iter/iters_per_epoch : 11/78 
+DLL 2023-07-12 20:13:33.663980 - (0, 11) train_loss : 0.001904733362607658 
+DLL 2023-07-12 20:13:34.175724 - (0, 11) train_items_per_sec : 88517.3361410073 items/s
+DLL 2023-07-12 20:13:34.175755 - (0, 11) train_iter_time : 0.7230222100001811 
+DLL 2023-07-12 20:13:34.175866 - (0, 12) glob_iter/iters_per_epoch : 12/78 
+DLL 2023-07-12 20:13:34.383794 - (0, 12) train_loss : 0.002156726084649563 
+DLL 2023-07-12 20:13:34.898928 - (0, 12) train_items_per_sec : 88512.0738130655 items/s
+DLL 2023-07-12 20:13:34.898961 - (0, 12) train_iter_time : 0.7230651959998795 
+DLL 2023-07-12 20:13:34.900630 - (0, 13) glob_iter/iters_per_epoch : 13/78 
+DLL 2023-07-12 20:13:35.110923 - (0, 13) train_loss : 0.00269884685985744 
+DLL 2023-07-12 20:13:35.625553 - (0, 13) train_items_per_sec : 88284.65826524586 items/s
+DLL 2023-07-12 20:13:35.625585 - (0, 13) train_iter_time : 0.7249277649998476 
+DLL 2023-07-12 20:13:35.625696 - (0, 14) glob_iter/iters_per_epoch : 14/78 
+DLL 2023-07-12 20:13:35.839426 - (0, 14) train_loss : 0.0015838425606489182 
+DLL 2023-07-12 20:13:36.352098 - (0, 14) train_items_per_sec : 88105.03357883285 items/s
+DLL 2023-07-12 20:13:36.352130 - (0, 14) train_iter_time : 0.72640571599959 
+DLL 2023-07-12 20:13:36.352245 - (0, 15) glob_iter/iters_per_epoch : 15/78 
+DLL 2023-07-12 20:13:36.557741 - (0, 15) train_loss : 0.001822038902901113 
+DLL 2023-07-12 20:13:37.075408 - (0, 15) train_items_per_sec : 88499.97023842705 items/s
+DLL 2023-07-12 20:13:37.075449 - (0, 15) train_iter_time : 0.7231640850000076 
+DLL 2023-07-12 20:13:37.075592 - (0, 16) glob_iter/iters_per_epoch : 16/78 
+DLL 2023-07-12 20:13:37.287150 - (0, 16) train_loss : 0.002946938155218959 
+DLL 2023-07-12 20:13:37.799293 - (0, 16) train_items_per_sec : 88433.96223306596 items/s
+DLL 2023-07-12 20:13:37.799327 - (0, 16) train_iter_time : 0.7237038619996383 
+DLL 2023-07-12 20:13:37.800392 - (0, 17) glob_iter/iters_per_epoch : 17/78 
+DLL 2023-07-12 20:13:38.001429 - (0, 17) train_loss : 0.0020861157681792974 
+DLL 2023-07-12 20:13:38.522356 - (0, 17) train_items_per_sec : 88646.18683325427 items/s
+DLL 2023-07-12 20:13:38.522387 - (0, 17) train_iter_time : 0.7219712689998232 
+DLL 2023-07-12 20:13:38.523557 - (0, 18) glob_iter/iters_per_epoch : 18/78 
+DLL 2023-07-12 20:13:38.735166 - (0, 18) train_loss : 0.0018607915844768286 
+DLL 2023-07-12 20:13:39.249726 - (0, 18) train_items_per_sec : 88133.18853799331 items/s
+DLL 2023-07-12 20:13:39.249759 - (0, 18) train_iter_time : 0.7261736590003238 
+DLL 2023-07-12 20:13:39.251121 - (0, 19) glob_iter/iters_per_epoch : 19/78 
+DLL 2023-07-12 20:13:39.462165 - (0, 19) train_loss : 0.002691973466426134 
+DLL 2023-07-12 20:13:39.977553 - (0, 19) train_items_per_sec : 88101.05911706809 items/s
+DLL 2023-07-12 20:13:39.977591 - (0, 19) train_iter_time : 0.7264384860000064 
+DLL 2023-07-12 20:13:39.979042 - (0, 20) glob_iter/iters_per_epoch : 20/78 
+DLL 2023-07-12 20:13:40.187710 - (0, 20) train_loss : 0.002063951687887311 
+DLL 2023-07-12 20:13:40.701233 - (0, 20) train_items_per_sec : 88618.52521502928 items/s
+DLL 2023-07-12 20:13:40.701266 - (0, 20) train_iter_time : 0.722196626999903 
+DLL 2023-07-12 20:13:40.702503 - (0, 21) glob_iter/iters_per_epoch : 21/78 
+DLL 2023-07-12 20:13:40.914694 - (0, 21) train_loss : 0.0014645917108282447 
+DLL 2023-07-12 20:13:41.430784 - (0, 21) train_items_per_sec : 87877.52778078323 items/s
+DLL 2023-07-12 20:13:41.430846 - (0, 21) train_iter_time : 0.7282863049999833 
+DLL 2023-07-12 20:13:41.431957 - (0, 22) glob_iter/iters_per_epoch : 22/78 
+DLL 2023-07-12 20:13:41.643257 - (0, 22) train_loss : 0.003002750687301159 
+DLL 2023-07-12 20:13:42.156361 - (0, 22) train_items_per_sec : 88347.81679605218 items/s
+DLL 2023-07-12 20:13:42.156396 - (0, 22) train_iter_time : 0.7244095249998281 
+DLL 2023-07-12 20:13:42.157573 - (0, 23) glob_iter/iters_per_epoch : 23/78 
+DLL 2023-07-12 20:13:42.362704 - (0, 23) train_loss : 0.002174938563257456 
+DLL 2023-07-12 20:13:42.880501 - (0, 23) train_items_per_sec : 88528.16514166942 items/s
+DLL 2023-07-12 20:13:42.880548 - (0, 23) train_iter_time : 0.7229337680000754 
+DLL 2023-07-12 20:13:42.881665 - (0, 24) glob_iter/iters_per_epoch : 24/78 
+DLL 2023-07-12 20:13:43.092644 - (0, 24) train_loss : 0.002497905632480979 
+DLL 2023-07-12 20:13:43.609769 - (0, 24) train_items_per_sec : 87898.81519095523 items/s
+DLL 2023-07-12 20:13:43.609800 - (0, 24) train_iter_time : 0.7281099280003218 
+DLL 2023-07-12 20:13:43.611002 - (0, 25) glob_iter/iters_per_epoch : 25/78 
+DLL 2023-07-12 20:13:43.825351 - (0, 25) train_loss : 0.0021909174975007772 
+DLL 2023-07-12 20:13:44.338794 - (0, 25) train_items_per_sec : 87936.4695062193 items/s
+DLL 2023-07-12 20:13:44.338848 - (0, 25) train_iter_time : 0.7277981519996501 
+DLL 2023-07-12 20:13:44.340028 - (0, 26) glob_iter/iters_per_epoch : 26/78 
+DLL 2023-07-12 20:13:44.545136 - (0, 26) train_loss : 0.0009575519943609834 
+DLL 2023-07-12 20:13:45.064885 - (0, 26) train_items_per_sec : 88292.76422681051 items/s
+DLL 2023-07-12 20:13:45.064920 - (0, 26) train_iter_time : 0.7248612110001886 
+DLL 2023-07-12 20:13:45.066530 - (0, 27) glob_iter/iters_per_epoch : 27/78 
+DLL 2023-07-12 20:13:45.278578 - (0, 27) train_loss : 0.0016052908031269908 
+DLL 2023-07-12 20:13:45.793063 - (0, 27) train_items_per_sec : 88088.975293553 items/s
+DLL 2023-07-12 20:13:45.793098 - (0, 27) train_iter_time : 0.7265381369998067 
+DLL 2023-07-12 20:13:45.794557 - (0, 28) glob_iter/iters_per_epoch : 28/78 
+DLL 2023-07-12 20:13:46.005960 - (0, 28) train_loss : 0.002770477905869484 
+DLL 2023-07-12 20:13:46.522461 - (0, 28) train_items_per_sec : 87922.80869761368 items/s
+DLL 2023-07-12 20:13:46.522494 - (0, 28) train_iter_time : 0.7279112320002241 
+DLL 2023-07-12 20:13:46.523623 - (0, 29) glob_iter/iters_per_epoch : 29/78 
+DLL 2023-07-12 20:13:46.734698 - (0, 29) train_loss : 0.0021189111284911633 
+DLL 2023-07-12 20:13:47.250307 - (0, 29) train_items_per_sec : 88070.6136859874 items/s
+DLL 2023-07-12 20:13:47.250341 - (0, 29) train_iter_time : 0.7266896109999834 
+DLL 2023-07-12 20:13:47.251581 - (0, 30) glob_iter/iters_per_epoch : 30/78 
+DLL 2023-07-12 20:13:47.463090 - (0, 30) train_loss : 0.004171343054622412 
+DLL 2023-07-12 20:13:47.977989 - (0, 30) train_items_per_sec : 88104.28511065443 items/s
+DLL 2023-07-12 20:13:47.978022 - (0, 30) train_iter_time : 0.7264118869998129 
+DLL 2023-07-12 20:13:47.979316 - (0, 31) glob_iter/iters_per_epoch : 31/78 
+DLL 2023-07-12 20:13:48.189135 - (0, 31) train_loss : 0.0027633963618427515 
+DLL 2023-07-12 20:13:48.705088 - (0, 31) train_items_per_sec : 88181.22123882112 items/s
+DLL 2023-07-12 20:13:48.705121 - (0, 31) train_iter_time : 0.7257781089997479 
+DLL 2023-07-12 20:13:48.706476 - (0, 32) glob_iter/iters_per_epoch : 32/78 
+DLL 2023-07-12 20:13:48.918537 - (0, 32) train_loss : 0.0019049713155254722 
+DLL 2023-07-12 20:13:49.433838 - (0, 32) train_items_per_sec : 87988.58334928525 items/s
+DLL 2023-07-12 20:13:49.433871 - (0, 32) train_iter_time : 0.7273670920003497 
+DLL 2023-07-12 20:13:49.435329 - (0, 33) glob_iter/iters_per_epoch : 33/78 
+DLL 2023-07-12 20:13:49.647336 - (0, 33) train_loss : 0.002339111641049385 
+DLL 2023-07-12 20:13:50.163534 - (0, 33) train_items_per_sec : 87886.80246854134 items/s
+DLL 2023-07-12 20:13:50.163569 - (0, 33) train_iter_time : 0.7282094490001327 
+DLL 2023-07-12 20:13:50.165137 - (0, 34) glob_iter/iters_per_epoch : 34/78 
+DLL 2023-07-12 20:13:50.372117 - (0, 34) train_loss : 0.0020714490674436092 
+DLL 2023-07-12 20:13:50.889405 - (0, 34) train_items_per_sec : 88364.20226051197 items/s
+DLL 2023-07-12 20:13:50.889438 - (0, 34) train_iter_time : 0.7242751970002246 
+DLL 2023-07-12 20:13:50.890855 - (0, 35) glob_iter/iters_per_epoch : 35/78 
+DLL 2023-07-12 20:13:51.102729 - (0, 35) train_loss : 0.00217114738188684 
+DLL 2023-07-12 20:13:51.619218 - (0, 35) train_items_per_sec : 87867.69879217535 items/s
+DLL 2023-07-12 20:13:51.619252 - (0, 35) train_iter_time : 0.7283677719997286 
+DLL 2023-07-12 20:13:51.620042 - (0, 36) glob_iter/iters_per_epoch : 36/78 
+DLL 2023-07-12 20:13:51.832858 - (0, 36) train_loss : 0.0038564386777579784 
+DLL 2023-07-12 20:13:52.348862 - (0, 36) train_items_per_sec : 87812.37033135632 items/s
+DLL 2023-07-12 20:13:52.348897 - (0, 36) train_iter_time : 0.7288266990003649 
+DLL 2023-07-12 20:13:52.350250 - (0, 37) glob_iter/iters_per_epoch : 37/78 
+DLL 2023-07-12 20:13:52.555078 - (0, 37) train_loss : 0.0015465328469872475 
+DLL 2023-07-12 20:13:53.073721 - (0, 37) train_items_per_sec : 88461.72014302829 items/s
+DLL 2023-07-12 20:13:53.073755 - (0, 37) train_iter_time : 0.7234767749996536 
+DLL 2023-07-12 20:13:53.075224 - (0, 38) glob_iter/iters_per_epoch : 38/78 
+DLL 2023-07-12 20:13:53.288817 - (0, 38) train_loss : 0.0019210026366636157 
+DLL 2023-07-12 20:13:53.804687 - (0, 38) train_items_per_sec : 87734.99037495781 items/s
+DLL 2023-07-12 20:13:53.804723 - (0, 38) train_iter_time : 0.7294695049999973 
+DLL 2023-07-12 20:13:53.806157 - (0, 39) glob_iter/iters_per_epoch : 39/78 
+DLL 2023-07-12 20:13:54.019915 - (0, 39) train_loss : 0.0014257453149184585 
+DLL 2023-07-12 20:13:54.535568 - (0, 39) train_items_per_sec : 87741.24233514494 items/s
+DLL 2023-07-12 20:13:54.535601 - (0, 39) train_iter_time : 0.7294175269998959 
+DLL 2023-07-12 20:13:54.536994 - (0, 40) glob_iter/iters_per_epoch : 40/78 
+DLL 2023-07-12 20:13:54.744517 - (0, 40) train_loss : 0.00208518048748374 
+DLL 2023-07-12 20:13:55.262669 - (0, 40) train_items_per_sec : 88192.96250886205 items/s
+DLL 2023-07-12 20:13:55.262702 - (0, 40) train_iter_time : 0.7256814850002229 
+DLL 2023-07-12 20:13:55.263835 - (0, 41) glob_iter/iters_per_epoch : 41/78 
+DLL 2023-07-12 20:13:55.476686 - (0, 41) train_loss : 0.002245573326945305 
+DLL 2023-07-12 20:13:55.992966 - (0, 41) train_items_per_sec : 87774.96737727024 items/s
+DLL 2023-07-12 20:13:55.993001 - (0, 41) train_iter_time : 0.7291372689996933 
+DLL 2023-07-12 20:13:55.994560 - (0, 42) glob_iter/iters_per_epoch : 42/78 
+DLL 2023-07-12 20:13:56.207710 - (0, 42) train_loss : 0.002042248146608472 
+DLL 2023-07-12 20:13:56.722945 - (0, 42) train_items_per_sec : 87864.90168392239 items/s
+DLL 2023-07-12 20:13:56.723005 - (0, 42) train_iter_time : 0.7283909590000803 
+DLL 2023-07-12 20:13:56.724100 - (0, 43) glob_iter/iters_per_epoch : 43/78 
+DLL 2023-07-12 20:13:56.934129 - (0, 43) train_loss : 0.002546385396271944 
+DLL 2023-07-12 20:13:57.452169 - (0, 43) train_items_per_sec : 87903.02001521 items/s
+DLL 2023-07-12 20:13:57.452202 - (0, 43) train_iter_time : 0.7280750990003071 
+DLL 2023-07-12 20:13:57.453397 - (0, 44) glob_iter/iters_per_epoch : 44/78 
+DLL 2023-07-12 20:13:57.665634 - (0, 44) train_loss : 0.002383449813351035 
+DLL 2023-07-12 20:13:58.180114 - (0, 44) train_items_per_sec : 88066.60451977249 items/s
+DLL 2023-07-12 20:13:58.180150 - (0, 44) train_iter_time : 0.7267226930002835 
+DLL 2023-07-12 20:13:58.181575 - (0, 45) glob_iter/iters_per_epoch : 45/78 
+DLL 2023-07-12 20:13:58.392722 - (0, 45) train_loss : 0.0020072893239557743 
+DLL 2023-07-12 20:13:58.908935 - (0, 45) train_items_per_sec : 87988.577179897 items/s
+DLL 2023-07-12 20:13:58.908966 - (0, 45) train_iter_time : 0.7273671430002651 
+DLL 2023-07-12 20:13:58.910223 - (0, 46) glob_iter/iters_per_epoch : 46/78 
+DLL 2023-07-12 20:13:59.121897 - (0, 46) train_loss : 0.002604077337309718 
+DLL 2023-07-12 20:13:59.639052 - (0, 46) train_items_per_sec : 87811.52875473512 items/s
+DLL 2023-07-12 20:13:59.639084 - (0, 46) train_iter_time : 0.7288336840001648 
+DLL 2023-07-12 20:13:59.640060 - (0, 47) glob_iter/iters_per_epoch : 47/78 
+DLL 2023-07-12 20:13:59.852524 - (0, 47) train_loss : 0.001263776677660644 
+DLL 2023-07-12 20:14:00.367905 - (0, 47) train_items_per_sec : 87930.06683275569 items/s
+DLL 2023-07-12 20:14:00.367939 - (0, 47) train_iter_time : 0.7278511469999103 
+DLL 2023-07-12 20:14:00.369310 - (0, 48) glob_iter/iters_per_epoch : 48/78 
+DLL 2023-07-12 20:14:00.578743 - (0, 48) train_loss : 0.0014681026805192232 
+DLL 2023-07-12 20:14:01.095690 - (0, 48) train_items_per_sec : 88107.33679610874 items/s
+DLL 2023-07-12 20:14:01.095721 - (0, 48) train_iter_time : 0.7263867269998627 
+DLL 2023-07-12 20:14:01.096990 - (0, 49) glob_iter/iters_per_epoch : 49/78 
+DLL 2023-07-12 20:14:01.311176 - (0, 49) train_loss : 0.002291016047820449 
+DLL 2023-07-12 20:14:01.827204 - (0, 49) train_items_per_sec : 87644.94059513268 items/s
+DLL 2023-07-12 20:14:01.827236 - (0, 49) train_iter_time : 0.7302189900001395 
+DLL 2023-07-12 20:14:01.828550 - (0, 50) glob_iter/iters_per_epoch : 50/78 
+DLL 2023-07-12 20:14:02.041626 - (0, 50) train_loss : 0.0029302830807864666 
+DLL 2023-07-12 20:14:02.557465 - (0, 50) train_items_per_sec : 87801.15212121839 items/s
+DLL 2023-07-12 20:14:02.557497 - (0, 50) train_iter_time : 0.728919820000101 
+DLL 2023-07-12 20:14:02.558799 - (0, 51) glob_iter/iters_per_epoch : 51/78 
+DLL 2023-07-12 20:14:02.767202 - (0, 51) train_loss : 0.0030032021459192038 
+DLL 2023-07-12 20:14:03.285697 - (0, 51) train_items_per_sec : 88044.82176985528 items/s
+DLL 2023-07-12 20:14:03.285731 - (0, 51) train_iter_time : 0.7269024879997232 
+DLL 2023-07-12 20:14:03.287168 - (0, 52) glob_iter/iters_per_epoch : 52/78 
+DLL 2023-07-12 20:14:03.499495 - (0, 52) train_loss : 0.002333219861611724 
+DLL 2023-07-12 20:14:04.016307 - (0, 52) train_items_per_sec : 87774.2021158673 items/s
+DLL 2023-07-12 20:14:04.016347 - (0, 52) train_iter_time : 0.7291436259997681 
+DLL 2023-07-12 20:14:04.017777 - (0, 53) glob_iter/iters_per_epoch : 53/78 
+DLL 2023-07-12 20:14:04.230616 - (0, 53) train_loss : 0.0018508475041016936 
+DLL 2023-07-12 20:14:04.749247 - (0, 53) train_items_per_sec : 87494.40261396758 items/s
+DLL 2023-07-12 20:14:04.749281 - (0, 53) train_iter_time : 0.7314753639998344 
+DLL 2023-07-12 20:14:04.750629 - (0, 54) glob_iter/iters_per_epoch : 54/78 
+DLL 2023-07-12 20:14:04.957995 - (0, 54) train_loss : 0.0021394987124949694 
+DLL 2023-07-12 20:14:05.477775 - (0, 54) train_items_per_sec : 88014.61611422255 items/s
+DLL 2023-07-12 20:14:05.477809 - (0, 54) train_iter_time : 0.727151952999975 
+DLL 2023-07-12 20:14:05.479168 - (0, 55) glob_iter/iters_per_epoch : 55/78 
+DLL 2023-07-12 20:14:05.691007 - (0, 55) train_loss : 0.0030705833341926336 
+DLL 2023-07-12 20:14:06.211851 - (0, 55) train_items_per_sec : 87349.51313277619 items/s
+DLL 2023-07-12 20:14:06.211889 - (0, 55) train_iter_time : 0.7326886860000741 
+DLL 2023-07-12 20:14:06.213181 - (0, 56) glob_iter/iters_per_epoch : 56/78 
+DLL 2023-07-12 20:14:06.426570 - (0, 56) train_loss : 0.002394376788288355 
+DLL 2023-07-12 20:14:06.941787 - (0, 56) train_items_per_sec : 87838.3990966839 items/s
+DLL 2023-07-12 20:14:06.941825 - (0, 56) train_iter_time : 0.7286107289996835 
+DLL 2023-07-12 20:14:06.943196 - (0, 57) glob_iter/iters_per_epoch : 57/78 
+DLL 2023-07-12 20:14:07.149666 - (0, 57) train_loss : 0.0026424205861985683 
+DLL 2023-07-12 20:14:07.670266 - (0, 57) train_items_per_sec : 88023.88792275084 items/s
+DLL 2023-07-12 20:14:07.670300 - (0, 57) train_iter_time : 0.7270753599996169 
+DLL 2023-07-12 20:14:07.671629 - (0, 58) glob_iter/iters_per_epoch : 58/78 
+DLL 2023-07-12 20:14:07.883787 - (0, 58) train_loss : 0.0023153743240982294 
+DLL 2023-07-12 20:14:08.402993 - (0, 58) train_items_per_sec : 87507.24980473104 items/s
+DLL 2023-07-12 20:14:08.403027 - (0, 58) train_iter_time : 0.7313679740000225 
+DLL 2023-07-12 20:14:08.404029 - (0, 59) glob_iter/iters_per_epoch : 59/78 
+DLL 2023-07-12 20:14:08.618494 - (0, 59) train_loss : 0.0024722458329051733 
+DLL 2023-07-12 20:14:09.133777 - (0, 59) train_items_per_sec : 87701.00845033805 items/s
+DLL 2023-07-12 20:14:09.133808 - (0, 59) train_iter_time : 0.7297521559999041 
+DLL 2023-07-12 20:14:09.135277 - (0, 60) glob_iter/iters_per_epoch : 60/78 
+DLL 2023-07-12 20:14:09.327133 - (0, 60) train_loss : 0.0015696348855271935 
+DLL 2023-07-12 20:14:09.860015 - (0, 60) train_items_per_sec : 88306.95342473201 items/s
+DLL 2023-07-12 20:14:09.860066 - (0, 60) train_iter_time : 0.7247447400000055 
+DLL 2023-07-12 20:14:09.861429 - (0, 61) glob_iter/iters_per_epoch : 61/78 
+DLL 2023-07-12 20:14:10.077139 - (0, 61) train_loss : 0.0022114766761660576 
+DLL 2023-07-12 20:14:10.594663 - (0, 61) train_items_per_sec : 87283.85112346022 items/s
+DLL 2023-07-12 20:14:10.594697 - (0, 61) train_iter_time : 0.7332398739999917 
+DLL 2023-07-12 20:14:10.595858 - (0, 62) glob_iter/iters_per_epoch : 62/78 
+DLL 2023-07-12 20:14:10.809071 - (0, 62) train_loss : 0.0017084142891690135 
+DLL 2023-07-12 20:14:11.329147 - (0, 62) train_items_per_sec : 87277.67786098046 items/s
+DLL 2023-07-12 20:14:11.329180 - (0, 62) train_iter_time : 0.733291736999945 
+DLL 2023-07-12 20:14:11.330443 - (0, 63) glob_iter/iters_per_epoch : 63/78 
+DLL 2023-07-12 20:14:11.538734 - (0, 63) train_loss : 0.0015121548203751445 
+DLL 2023-07-12 20:14:12.056521 - (0, 63) train_items_per_sec : 88144.06225870657 items/s
+DLL 2023-07-12 20:14:12.056554 - (0, 63) train_iter_time : 0.7260840760000065 
+DLL 2023-07-12 20:14:12.057943 - (0, 64) glob_iter/iters_per_epoch : 64/78 
+DLL 2023-07-12 20:14:12.272495 - (0, 64) train_loss : 0.002002851804718375 
+DLL 2023-07-12 20:14:12.791837 - (0, 64) train_items_per_sec : 87205.44529878064 items/s
+DLL 2023-07-12 20:14:12.791873 - (0, 64) train_iter_time : 0.7338991249998799 
+DLL 2023-07-12 20:14:12.793276 - (0, 65) glob_iter/iters_per_epoch : 65/78 
+DLL 2023-07-12 20:14:13.007763 - (0, 65) train_loss : 0.001212309580296278 
+DLL 2023-07-12 20:14:13.525512 - (0, 65) train_items_per_sec : 87402.85737451045 items/s
+DLL 2023-07-12 20:14:13.525546 - (0, 65) train_iter_time : 0.7322415069997987 
+DLL 2023-07-12 20:14:13.526967 - (0, 66) glob_iter/iters_per_epoch : 66/78 
+DLL 2023-07-12 20:14:13.735189 - (0, 66) train_loss : 0.0028264187276363373 
+DLL 2023-07-12 20:14:14.253850 - (0, 66) train_items_per_sec : 88046.35521735172 items/s
+DLL 2023-07-12 20:14:14.253882 - (0, 66) train_iter_time : 0.7268898280003668 
+DLL 2023-07-12 20:14:14.255080 - (0, 67) glob_iter/iters_per_epoch : 67/78 
+DLL 2023-07-12 20:14:14.469781 - (0, 67) train_loss : 0.0021959298755973577 
+DLL 2023-07-12 20:14:14.987641 - (0, 67) train_items_per_sec : 87364.1406221865 items/s
+DLL 2023-07-12 20:14:14.987676 - (0, 67) train_iter_time : 0.7325660109995624 
+DLL 2023-07-12 20:14:14.989319 - (0, 68) glob_iter/iters_per_epoch : 68/78 
+DLL 2023-07-12 20:14:15.203294 - (0, 68) train_loss : 0.0015247870469465852 
+DLL 2023-07-12 20:14:15.719995 - (0, 68) train_items_per_sec : 87589.46595050677 items/s
+DLL 2023-07-12 20:14:15.720032 - (0, 68) train_iter_time : 0.7306814730000042 
+DLL 2023-07-12 20:14:15.721493 - (0, 69) glob_iter/iters_per_epoch : 69/78 
+DLL 2023-07-12 20:14:15.932313 - (0, 69) train_loss : 0.0037806625477969646 
+DLL 2023-07-12 20:14:16.450531 - (0, 69) train_items_per_sec : 87786.22055962819 items/s
+DLL 2023-07-12 20:14:16.450564 - (0, 69) train_iter_time : 0.7290438019999783 
+DLL 2023-07-12 20:14:16.451737 - (0, 70) glob_iter/iters_per_epoch : 70/78 
+DLL 2023-07-12 20:14:16.666433 - (0, 70) train_loss : 0.0036126819904893637 
+DLL 2023-07-12 20:14:17.184707 - (0, 70) train_items_per_sec : 87315.14068166025 items/s
+DLL 2023-07-12 20:14:17.184744 - (0, 70) train_iter_time : 0.7329771160002565 
+DLL 2023-07-12 20:14:17.186225 - (0, 71) glob_iter/iters_per_epoch : 71/78 
+DLL 2023-07-12 20:14:17.400231 - (0, 71) train_loss : 0.002572603989392519 
+DLL 2023-07-12 20:14:17.915969 - (0, 71) train_items_per_sec : 87701.16949071275 items/s
+DLL 2023-07-12 20:14:17.916015 - (0, 71) train_iter_time : 0.7297508159999779 
+DLL 2023-07-12 20:14:17.917304 - (0, 72) glob_iter/iters_per_epoch : 72/78 
+DLL 2023-07-12 20:14:18.127723 - (0, 72) train_loss : 0.0018606267403811216 
+DLL 2023-07-12 20:14:18.649533 - (0, 72) train_items_per_sec : 87403.81253105077 items/s
+DLL 2023-07-12 20:14:18.649568 - (0, 72) train_iter_time : 0.7322335050002948 
+DLL 2023-07-12 20:14:18.650629 - (0, 73) glob_iter/iters_per_epoch : 73/78 
+DLL 2023-07-12 20:14:18.863542 - (0, 73) train_loss : 0.0020415629260241985 
+DLL 2023-07-12 20:14:19.379397 - (0, 73) train_items_per_sec : 87818.8034737471 items/s
+DLL 2023-07-12 20:14:19.379429 - (0, 73) train_iter_time : 0.7287733090001893 
+DLL 2023-07-12 20:14:19.380550 - (0, 74) glob_iter/iters_per_epoch : 74/78 
+DLL 2023-07-12 20:14:19.594531 - (0, 74) train_loss : 0.002890064613893628 
+DLL 2023-07-12 20:14:20.113079 - (0, 74) train_items_per_sec : 87367.96264096192 items/s
+DLL 2023-07-12 20:14:20.113117 - (0, 74) train_iter_time : 0.7325339640001403 
+DLL 2023-07-12 20:14:20.114211 - (0, 75) glob_iter/iters_per_epoch : 75/78 
+DLL 2023-07-12 20:14:20.325502 - (0, 75) train_loss : 0.0026142869610339403 
+DLL 2023-07-12 20:14:20.846449 - (0, 75) train_items_per_sec : 87402.43518805977 items/s
+DLL 2023-07-12 20:14:20.846483 - (0, 75) train_iter_time : 0.7322450440001376 
+DLL 2023-07-12 20:14:20.847688 - (0, 76) glob_iter/iters_per_epoch : 76/78 
+DLL 2023-07-12 20:14:21.062050 - (0, 76) train_loss : 0.001208981266245246 
+DLL 2023-07-12 20:14:21.578463 - (0, 76) train_items_per_sec : 87577.67376779497 items/s
+DLL 2023-07-12 20:14:21.578511 - (0, 76) train_iter_time : 0.7307798580000053 
+DLL 2023-07-12 20:14:21.579925 - (0, 77) glob_iter/iters_per_epoch : 77/78 
+DLL 2023-07-12 20:14:21.793782 - (0, 77) train_loss : 0.0018688308773562312 
+DLL 2023-07-12 20:14:22.311712 - (0, 77) train_items_per_sec : 87456.57944807202 items/s
+DLL 2023-07-12 20:14:22.311748 - (0, 77) train_iter_time : 0.7317917120003585 
+DLL 2023-07-12 20:14:22.346995 - (0,) train_items_per_sec : 87165.09925057506 items/s
+DLL 2023-07-12 20:14:22.347039 - (0,) train_loss : 0.0018688308773562312 
+DLL 2023-07-12 20:14:22.347051 - (0,) train_epoch_time : 59.26145170500013 
+DLL 2023-07-12 20:14:22.695205 - (0, 78, 0) val_items_per_sec : 206807.85009893368 items/s
+DLL 2023-07-12 20:14:22.905440 - (0, 78, 1) val_items_per_sec : 306039.2469987373 items/s
+DLL 2023-07-12 20:14:23.116614 - (0, 78, 2) val_items_per_sec : 304527.5881841242 items/s
+DLL 2023-07-12 20:14:23.327606 - (0, 78, 3) val_items_per_sec : 304529.9472019087 items/s
+DLL 2023-07-12 20:14:23.542254 - (0, 78, 4) val_items_per_sec : 299501.48072147445 items/s
+DLL 2023-07-12 20:14:23.760125 - (0, 78, 5) val_items_per_sec : 295350.36292162526 items/s
+DLL 2023-07-12 20:14:23.974161 - (0, 78, 6) val_items_per_sec : 300670.7551128807 items/s
+DLL 2023-07-12 20:14:24.187060 - (0, 78, 7) val_items_per_sec : 302014.94508687983 items/s
+DLL 2023-07-12 20:14:24.398408 - (0, 78, 8) val_items_per_sec : 303831.1253434332 items/s
+DLL 2023-07-12 20:14:24.612601 - (0, 78, 9) val_items_per_sec : 299998.64203732257 items/s
+DLL 2023-07-12 20:14:24.829784 - (0, 78, 10) val_items_per_sec : 295891.3840053445 items/s
+DLL 2023-07-12 20:14:25.043736 - (0, 78, 11) val_items_per_sec : 300400.21945718967 items/s
+DLL 2023-07-12 20:14:25.929007 - (0, 78, 12) val_items_per_sec : 36191.07734797329 items/s
+DLL 2023-07-12 20:14:25.945164 - (0,) val_loss : 0.001951702073431359 None
+DLL 2023-07-12 20:14:25.945209 - (0,) val_items_per_sec : 273519.58650137135 items/s
+Saving model and optimizer state at epoch 0 to ./checkpoint_WaveGlow_0.pt
+Updating symlink ./checkpoint_WaveGlow_last.pt to point to checkpoint_WaveGlow_0.pt
+DLL 2023-07-12 20:14:29.034893 - (1, 0) glob_iter/iters_per_epoch : 78/78 
+DLL 2023-07-12 20:14:29.250440 - (1, 0) train_loss : 0.0014083783607929945 
+DLL 2023-07-12 20:14:29.762638 - (1, 0) train_items_per_sec : 87937.46366839905 items/s
+DLL 2023-07-12 20:14:29.762680 - (1, 0) train_iter_time : 0.7277899240002625 
+DLL 2023-07-12 20:14:29.763878 - (1, 1) glob_iter/iters_per_epoch : 79/78 
+DLL 2023-07-12 20:14:29.969314 - (1, 1) train_loss : 0.0018605744699016213 
+DLL 2023-07-12 20:14:30.492476 - (1, 1) train_items_per_sec : 87839.3457126079 items/s
+DLL 2023-07-12 20:14:30.492513 - (1, 1) train_iter_time : 0.7286028769999575 
+DLL 2023-07-12 20:14:30.494142 - (1, 2) glob_iter/iters_per_epoch : 80/78 
+DLL 2023-07-12 20:14:30.706324 - (1, 2) train_loss : 0.0020842382218688726 
+DLL 2023-07-12 20:14:31.221945 - (1, 2) train_items_per_sec : 87935.23360097995 items/s
+DLL 2023-07-12 20:14:31.221982 - (1, 2) train_iter_time : 0.727808381000159 
+DLL 2023-07-12 20:14:31.223192 - (1, 3) glob_iter/iters_per_epoch : 81/78 
+DLL 2023-07-12 20:14:31.436214 - (1, 3) train_loss : 0.0013421019539237022 
+DLL 2023-07-12 20:14:31.954796 - (1, 3) train_items_per_sec : 87478.46580881184 items/s
+DLL 2023-07-12 20:14:31.954830 - (1, 3) train_iter_time : 0.7316086239998185 
+DLL 2023-07-12 20:14:31.957957 - (1, 4) glob_iter/iters_per_epoch : 82/78 
+DLL 2023-07-12 20:14:32.167307 - (1, 4) train_loss : 0.003313993802294135 
+DLL 2023-07-12 20:14:32.682971 - (1, 4) train_items_per_sec : 88273.59142867468 items/s
+DLL 2023-07-12 20:14:32.683016 - (1, 4) train_iter_time : 0.7250186490000488 
+DLL 2023-07-12 20:14:32.683148 - (1, 5) glob_iter/iters_per_epoch : 83/78 
+DLL 2023-07-12 20:14:32.897786 - (1, 5) train_loss : 0.0016292164800688624 
+DLL 2023-07-12 20:14:33.416954 - (1, 5) train_items_per_sec : 87215.93130682802 items/s
+DLL 2023-07-12 20:14:33.416990 - (1, 5) train_iter_time : 0.7338108879998799 
+DLL 2023-07-12 20:14:33.420735 - (1, 6) glob_iter/iters_per_epoch : 84/78 
+DLL 2023-07-12 20:14:33.633924 - (1, 6) train_loss : 0.0029616625979542732 
+DLL 2023-07-12 20:14:34.149558 - (1, 6) train_items_per_sec : 87812.14984518015 items/s
+DLL 2023-07-12 20:14:34.149593 - (1, 6) train_iter_time : 0.7288285290001113 
+DLL 2023-07-12 20:14:34.150982 - (1, 7) glob_iter/iters_per_epoch : 85/78 
+DLL 2023-07-12 20:14:34.359545 - (1, 7) train_loss : 0.002675196621567011 
+DLL 2023-07-12 20:14:34.880144 - (1, 7) train_items_per_sec : 87771.34547455539 items/s
+DLL 2023-07-12 20:14:34.880181 - (1, 7) train_iter_time : 0.7291673569998238 
+DLL 2023-07-12 20:14:34.880296 - (1, 8) glob_iter/iters_per_epoch : 86/78 
+DLL 2023-07-12 20:14:35.093582 - (1, 8) train_loss : 0.0019783801399171352 
+DLL 2023-07-12 20:14:35.610361 - (1, 8) train_items_per_sec : 87662.90093825242 items/s
+DLL 2023-07-12 20:14:35.610394 - (1, 8) train_iter_time : 0.7300693830002274 
+DLL 2023-07-12 20:14:35.610516 - (1, 9) glob_iter/iters_per_epoch : 87/78 
+DLL 2023-07-12 20:14:35.825995 - (1, 9) train_loss : 0.0024432705249637365 
+DLL 2023-07-12 20:14:36.342519 - (1, 9) train_items_per_sec : 87430.4063016677 items/s
+DLL 2023-07-12 20:14:36.342553 - (1, 9) train_iter_time : 0.7320107809996443 
+DLL 2023-07-12 20:14:36.342675 - (1, 10) glob_iter/iters_per_epoch : 88/78 
+DLL 2023-07-12 20:14:36.549143 - (1, 10) train_loss : 0.0015703552635386586 
+DLL 2023-07-12 20:14:37.072250 - (1, 10) train_items_per_sec : 87721.96549352529 items/s
+DLL 2023-07-12 20:14:37.072286 - (1, 10) train_iter_time : 0.7295778160000737 
+DLL 2023-07-12 20:14:37.072438 - (1, 11) glob_iter/iters_per_epoch : 89/78 
+DLL 2023-07-12 20:14:37.286792 - (1, 11) train_loss : 0.0015427239704877138 
+DLL 2023-07-12 20:14:37.802382 - (1, 11) train_items_per_sec : 87677.58577838002 items/s
+DLL 2023-07-12 20:14:37.802416 - (1, 11) train_iter_time : 0.7299471060000542 
+DLL 2023-07-12 20:14:37.802532 - (1, 12) glob_iter/iters_per_epoch : 90/78 
+DLL 2023-07-12 20:14:38.015914 - (1, 12) train_loss : 0.0017397507326677442 
+DLL 2023-07-12 20:14:38.536575 - (1, 12) train_items_per_sec : 87187.94579349027 items/s
+DLL 2023-07-12 20:14:38.536612 - (1, 12) train_iter_time : 0.7340464260000772 
+DLL 2023-07-12 20:14:38.536732 - (1, 13) glob_iter/iters_per_epoch : 91/78 
+DLL 2023-07-12 20:14:38.744611 - (1, 13) train_loss : 0.0020958746317774057 
+DLL 2023-07-12 20:14:39.261218 - (1, 13) train_items_per_sec : 88338.09083183492 items/s
+DLL 2023-07-12 20:14:39.261254 - (1, 13) train_iter_time : 0.7244892819999222 
+DLL 2023-07-12 20:14:39.261373 - (1, 14) glob_iter/iters_per_epoch : 92/78 
+DLL 2023-07-12 20:14:39.473750 - (1, 14) train_loss : 0.0014113473007455468 
+DLL 2023-07-12 20:14:39.995295 - (1, 14) train_items_per_sec : 87202.27481090488 items/s
+DLL 2023-07-12 20:14:39.995335 - (1, 14) train_iter_time : 0.7339258079996398 
+DLL 2023-07-12 20:14:39.996769 - (1, 15) glob_iter/iters_per_epoch : 93/78 
+DLL 2023-07-12 20:14:40.210720 - (1, 15) train_loss : 0.0018854662775993347 
+DLL 2023-07-12 20:14:40.725188 - (1, 15) train_items_per_sec : 87860.77010172968 items/s
+DLL 2023-07-12 20:14:40.725225 - (1, 15) train_iter_time : 0.7284252110002853 
+DLL 2023-07-12 20:14:40.725340 - (1, 16) glob_iter/iters_per_epoch : 94/78 
+DLL 2023-07-12 20:14:40.931936 - (1, 16) train_loss : 0.001033363281749189 
+DLL 2023-07-12 20:14:41.456985 - (1, 16) train_items_per_sec : 87473.69127935354 items/s
+DLL 2023-07-12 20:14:41.457022 - (1, 16) train_iter_time : 0.7316485569999713 
+DLL 2023-07-12 20:14:41.458856 - (1, 17) glob_iter/iters_per_epoch : 95/78 
+DLL 2023-07-12 20:14:41.672716 - (1, 17) train_loss : 0.0022882965859025717 
+DLL 2023-07-12 20:14:42.188614 - (1, 17) train_items_per_sec : 87699.71029417013 items/s
+DLL 2023-07-12 20:14:42.188646 - (1, 17) train_iter_time : 0.7297629579998102 
+DLL 2023-07-12 20:14:42.190207 - (1, 18) glob_iter/iters_per_epoch : 96/78 
+DLL 2023-07-12 20:14:42.404751 - (1, 18) train_loss : 0.0017073573544621468 
+DLL 2023-07-12 20:14:42.924606 - (1, 18) train_items_per_sec : 87145.5599954498 items/s
+DLL 2023-07-12 20:14:42.924641 - (1, 18) train_iter_time : 0.734403451000162 
+DLL 2023-07-12 20:14:42.925935 - (1, 19) glob_iter/iters_per_epoch : 97/78 
+DLL 2023-07-12 20:14:43.129890 - (1, 19) train_loss : 0.002106072846800089 
+DLL 2023-07-12 20:14:43.652429 - (1, 19) train_items_per_sec : 88093.66771748432 items/s
+DLL 2023-07-12 20:14:43.652465 - (1, 19) train_iter_time : 0.7264994369998021 
+DLL 2023-07-12 20:14:43.653850 - (1, 20) glob_iter/iters_per_epoch : 98/78 
+DLL 2023-07-12 20:14:43.868151 - (1, 20) train_loss : 0.002918446669355035 
+DLL 2023-07-12 20:14:44.386398 - (1, 20) train_items_per_sec : 87365.5005388912 items/s
+DLL 2023-07-12 20:14:44.386444 - (1, 20) train_iter_time : 0.7325546080001004 
+DLL 2023-07-12 20:14:44.387750 - (1, 21) glob_iter/iters_per_epoch : 99/78 
+DLL 2023-07-12 20:14:44.601248 - (1, 21) train_loss : 0.002760910429060459 
+DLL 2023-07-12 20:14:45.119217 - (1, 21) train_items_per_sec : 87494.7412413382 items/s
+DLL 2023-07-12 20:14:45.119284 - (1, 21) train_iter_time : 0.7314725330002148 
+DLL 2023-07-12 20:14:45.120438 - (1, 22) glob_iter/iters_per_epoch : 100/78 
+DLL 2023-07-12 20:14:45.327971 - (1, 22) train_loss : 0.0019587096758186817 
+DLL 2023-07-12 20:14:45.848879 - (1, 22) train_items_per_sec : 87858.05219285366 items/s
+DLL 2023-07-12 20:14:45.848913 - (1, 22) train_iter_time : 0.7284477450002669 
+DLL 2023-07-12 20:14:45.850385 - (1, 23) glob_iter/iters_per_epoch : 101/78 
+DLL 2023-07-12 20:14:46.064888 - (1, 23) train_loss : 0.002017648657783866 
+DLL 2023-07-12 20:14:46.582520 - (1, 23) train_items_per_sec : 87415.0149175704 items/s
+DLL 2023-07-12 20:14:46.582556 - (1, 23) train_iter_time : 0.7321396680004 
+DLL 2023-07-12 20:14:46.583809 - (1, 24) glob_iter/iters_per_epoch : 102/78 
+DLL 2023-07-12 20:14:46.800000 - (1, 24) train_loss : 0.0014541336568072438 
+DLL 2023-07-12 20:14:47.317268 - (1, 24) train_items_per_sec : 87257.09183311016 items/s
+DLL 2023-07-12 20:14:47.317304 - (1, 24) train_iter_time : 0.7334647379998387 
+DLL 2023-07-12 20:14:47.318318 - (1, 25) glob_iter/iters_per_epoch : 103/78 
+DLL 2023-07-12 20:14:47.524879 - (1, 25) train_loss : 0.0023119228426367044 
+DLL 2023-07-12 20:14:48.046958 - (1, 25) train_items_per_sec : 87834.16561491136 items/s
+DLL 2023-07-12 20:14:48.047012 - (1, 25) train_iter_time : 0.7286458469998252 
+DLL 2023-07-12 20:14:48.048265 - (1, 26) glob_iter/iters_per_epoch : 104/78 
+DLL 2023-07-12 20:14:48.262469 - (1, 26) train_loss : 0.001847947365604341 
+DLL 2023-07-12 20:14:48.779443 - (1, 26) train_items_per_sec : 87529.24054462796 items/s
+DLL 2023-07-12 20:14:48.779480 - (1, 26) train_iter_time : 0.7311842260000958 
+DLL 2023-07-12 20:14:48.780445 - (1, 27) glob_iter/iters_per_epoch : 105/78 
+DLL 2023-07-12 20:14:48.995461 - (1, 27) train_loss : 0.002606530673801899 
+DLL 2023-07-12 20:14:49.515760 - (1, 27) train_items_per_sec : 87036.61256985884 items/s
+DLL 2023-07-12 20:14:49.515796 - (1, 27) train_iter_time : 0.7353227349999543 
+DLL 2023-07-12 20:14:49.517095 - (1, 28) glob_iter/iters_per_epoch : 106/78 
+DLL 2023-07-12 20:14:49.723794 - (1, 28) train_loss : 0.0030029453337192535 
+DLL 2023-07-12 20:14:50.244082 - (1, 28) train_items_per_sec : 88033.94648398845 items/s
+DLL 2023-07-12 20:14:50.244121 - (1, 28) train_iter_time : 0.7269922860000406 
+DLL 2023-07-12 20:14:50.245788 - (1, 29) glob_iter/iters_per_epoch : 107/78 
+DLL 2023-07-12 20:14:50.459925 - (1, 29) train_loss : 0.002286092145368457 
+DLL 2023-07-12 20:14:50.977474 - (1, 29) train_items_per_sec : 87468.616140905 items/s
+DLL 2023-07-12 20:14:50.977511 - (1, 29) train_iter_time : 0.7316910090003148 
+DLL 2023-07-12 20:14:50.978836 - (1, 30) glob_iter/iters_per_epoch : 108/78 
+DLL 2023-07-12 20:14:51.193753 - (1, 30) train_loss : 0.0013558822683990002 
+DLL 2023-07-12 20:14:51.713105 - (1, 30) train_items_per_sec : 87160.88013350009 items/s
+DLL 2023-07-12 20:14:51.713151 - (1, 30) train_iter_time : 0.7342743659996813 
+DLL 2023-07-12 20:14:51.714536 - (1, 31) glob_iter/iters_per_epoch : 109/78 
+DLL 2023-07-12 20:14:51.922357 - (1, 31) train_loss : 0.0023844956886023283 
+DLL 2023-07-12 20:14:52.442249 - (1, 31) train_items_per_sec : 87946.04136589605 items/s
+DLL 2023-07-12 20:14:52.442284 - (1, 31) train_iter_time : 0.7277189400001589 
+DLL 2023-07-12 20:14:52.443534 - (1, 32) glob_iter/iters_per_epoch : 110/78 
+DLL 2023-07-12 20:14:52.657208 - (1, 32) train_loss : 0.002284169429913163 
+DLL 2023-07-12 20:14:53.178294 - (1, 32) train_items_per_sec : 87102.62337181358 items/s
+DLL 2023-07-12 20:14:53.178327 - (1, 32) train_iter_time : 0.7347654699997292 
+DLL 2023-07-12 20:14:53.179692 - (1, 33) glob_iter/iters_per_epoch : 111/78 
+DLL 2023-07-12 20:14:53.394744 - (1, 33) train_loss : 0.0024101599119603634 
+DLL 2023-07-12 20:14:53.910913 - (1, 33) train_items_per_sec : 87524.27829458845 items/s
+DLL 2023-07-12 20:14:53.910949 - (1, 33) train_iter_time : 0.731225681000069 
+DLL 2023-07-12 20:14:53.912090 - (1, 34) glob_iter/iters_per_epoch : 112/78 
+DLL 2023-07-12 20:14:54.121211 - (1, 34) train_loss : 0.0019503836520016193 
+DLL 2023-07-12 20:14:54.643905 - (1, 34) train_items_per_sec : 87453.0547561154 items/s
+DLL 2023-07-12 20:14:54.643938 - (1, 34) train_iter_time : 0.7318212059999496 
+DLL 2023-07-12 20:14:54.645368 - (1, 35) glob_iter/iters_per_epoch : 113/78 
+DLL 2023-07-12 20:14:54.859846 - (1, 35) train_loss : 0.001250579603947699 
+DLL 2023-07-12 20:14:55.376314 - (1, 35) train_items_per_sec : 87557.164482286 items/s
+DLL 2023-07-12 20:14:55.376347 - (1, 35) train_iter_time : 0.7309510350000892 
+DLL 2023-07-12 20:14:55.377696 - (1, 36) glob_iter/iters_per_epoch : 114/78 
+DLL 2023-07-12 20:14:55.592797 - (1, 36) train_loss : 0.002147702733054757 
+DLL 2023-07-12 20:14:56.111613 - (1, 36) train_items_per_sec : 87202.82303227704 items/s
+DLL 2023-07-12 20:14:56.111646 - (1, 36) train_iter_time : 0.7339211939997767 
+DLL 2023-07-12 20:14:56.113361 - (1, 37) glob_iter/iters_per_epoch : 115/78 
+DLL 2023-07-12 20:14:56.321630 - (1, 37) train_loss : 0.0014776408206671476 
+DLL 2023-07-12 20:14:56.841160 - (1, 37) train_items_per_sec : 87935.7542259226 items/s
+DLL 2023-07-12 20:14:56.841196 - (1, 37) train_iter_time : 0.7278040719997989 
+DLL 2023-07-12 20:14:56.842651 - (1, 38) glob_iter/iters_per_epoch : 116/78 
+DLL 2023-07-12 20:14:57.056925 - (1, 38) train_loss : 0.001625965116545558 
+DLL 2023-07-12 20:14:57.576161 - (1, 38) train_items_per_sec : 87251.03238284148 items/s
+DLL 2023-07-12 20:14:57.576193 - (1, 38) train_iter_time : 0.7335156760000245 
+DLL 2023-07-12 20:14:57.577379 - (1, 39) glob_iter/iters_per_epoch : 117/78 
+DLL 2023-07-12 20:14:57.793529 - (1, 39) train_loss : 0.0025585212279111147 
+DLL 2023-07-12 20:14:58.310287 - (1, 39) train_items_per_sec : 87322.84904006579 items/s
+DLL 2023-07-12 20:14:58.310322 - (1, 39) train_iter_time : 0.7329124130001219 
+DLL 2023-07-12 20:14:58.311681 - (1, 40) glob_iter/iters_per_epoch : 118/78 
+DLL 2023-07-12 20:14:58.519184 - (1, 40) train_loss : 0.0022984808310866356 
+DLL 2023-07-12 20:14:59.041969 - (1, 40) train_items_per_sec : 87636.05840400133 items/s
+DLL 2023-07-12 20:14:59.042004 - (1, 40) train_iter_time : 0.7302929999996195 
+DLL 2023-07-12 20:14:59.043354 - (1, 41) glob_iter/iters_per_epoch : 119/78 
+DLL 2023-07-12 20:14:59.257394 - (1, 41) train_loss : 0.0013256656238809228 
+DLL 2023-07-12 20:14:59.775191 - (1, 41) train_items_per_sec : 87450.2078671027 items/s
+DLL 2023-07-12 20:14:59.775229 - (1, 41) train_iter_time : 0.7318450299999313 
+DLL 2023-07-12 20:14:59.776149 - (1, 42) glob_iter/iters_per_epoch : 120/78 
+DLL 2023-07-12 20:14:59.991529 - (1, 42) train_loss : 0.0021141646429896355 
+DLL 2023-07-12 20:15:00.511094 - (1, 42) train_items_per_sec : 87080.69537954107 items/s
+DLL 2023-07-12 20:15:00.511165 - (1, 42) train_iter_time : 0.7349504930002695 
+DLL 2023-07-12 20:15:00.512451 - (1, 43) glob_iter/iters_per_epoch : 121/78 
+DLL 2023-07-12 20:15:00.721290 - (1, 43) train_loss : 0.002180635230615735 
+DLL 2023-07-12 20:15:01.242061 - (1, 43) train_items_per_sec : 87717.22506052823 items/s
+DLL 2023-07-12 20:15:01.242095 - (1, 43) train_iter_time : 0.7296172440001101 
+DLL 2023-07-12 20:15:01.243295 - (1, 44) glob_iter/iters_per_epoch : 122/78 
+DLL 2023-07-12 20:15:01.457429 - (1, 44) train_loss : 0.0015087061328813434 
+DLL 2023-07-12 20:15:01.977588 - (1, 44) train_items_per_sec : 87157.9982283273 items/s
+DLL 2023-07-12 20:15:01.977625 - (1, 44) train_iter_time : 0.7342986450003082 
+DLL 2023-07-12 20:15:01.978795 - (1, 45) glob_iter/iters_per_epoch : 123/78 
+DLL 2023-07-12 20:15:02.195109 - (1, 45) train_loss : 0.002541069407016039 
+DLL 2023-07-12 20:15:02.712354 - (1, 45) train_items_per_sec : 87244.59901914398 items/s
+DLL 2023-07-12 20:15:02.712387 - (1, 45) train_iter_time : 0.7335697650000839 
+DLL 2023-07-12 20:15:02.713798 - (1, 46) glob_iter/iters_per_epoch : 124/78 
+DLL 2023-07-12 20:15:02.921214 - (1, 46) train_loss : 0.0023287893272936344 
+DLL 2023-07-12 20:15:03.444435 - (1, 46) train_items_per_sec : 87594.26655449509 items/s
+DLL 2023-07-12 20:15:03.444468 - (1, 46) train_iter_time : 0.7306414280001263 
+DLL 2023-07-12 20:15:03.445759 - (1, 47) glob_iter/iters_per_epoch : 125/78 
+DLL 2023-07-12 20:15:03.659841 - (1, 47) train_loss : 0.0017270779935643077 
+DLL 2023-07-12 20:15:04.176386 - (1, 47) train_items_per_sec : 87595.42755369344 items/s
+DLL 2023-07-12 20:15:04.176422 - (1, 47) train_iter_time : 0.7306317440002204 
+DLL 2023-07-12 20:15:04.177795 - (1, 48) glob_iter/iters_per_epoch : 126/78 
+DLL 2023-07-12 20:15:04.393615 - (1, 48) train_loss : 0.002598671242594719 
+DLL 2023-07-12 20:15:04.915008 - (1, 48) train_items_per_sec : 86812.84047261573 items/s
+DLL 2023-07-12 20:15:04.915047 - (1, 48) train_iter_time : 0.7372181309997359 
+DLL 2023-07-12 20:15:04.916029 - (1, 49) glob_iter/iters_per_epoch : 127/78 
+DLL 2023-07-12 20:15:05.111621 - (1, 49) train_loss : 0.002707742853090167 
+DLL 2023-07-12 20:15:05.639685 - (1, 49) train_items_per_sec : 88439.25878393103 items/s
+DLL 2023-07-12 20:15:05.639721 - (1, 49) train_iter_time : 0.7236605200000668 
+DLL 2023-07-12 20:15:05.641224 - (1, 50) glob_iter/iters_per_epoch : 128/78 
+DLL 2023-07-12 20:15:05.856510 - (1, 50) train_loss : 0.00264070276170969 
+DLL 2023-07-12 20:15:06.373977 - (1, 50) train_items_per_sec : 87341.23972618938 items/s
+DLL 2023-07-12 20:15:06.374010 - (1, 50) train_iter_time : 0.7327580900000612 
+DLL 2023-07-12 20:15:06.375334 - (1, 51) glob_iter/iters_per_epoch : 129/78 
+DLL 2023-07-12 20:15:06.588822 - (1, 51) train_loss : 0.0010090357391163707 
+DLL 2023-07-12 20:15:07.110133 - (1, 51) train_items_per_sec : 87097.75963181339 items/s
+DLL 2023-07-12 20:15:07.110171 - (1, 51) train_iter_time : 0.7348065010000937 
+DLL 2023-07-12 20:15:07.111378 - (1, 52) glob_iter/iters_per_epoch : 130/78 
+DLL 2023-07-12 20:15:07.319691 - (1, 52) train_loss : 0.0019114085007458925 
+DLL 2023-07-12 20:15:07.839536 - (1, 52) train_items_per_sec : 87892.35547461476 items/s
+DLL 2023-07-12 20:15:07.839572 - (1, 52) train_iter_time : 0.7281634410001061 
+DLL 2023-07-12 20:15:07.840959 - (1, 53) glob_iter/iters_per_epoch : 131/78 
+DLL 2023-07-12 20:15:08.055219 - (1, 53) train_loss : 0.0017808331176638603 
+DLL 2023-07-12 20:15:08.574560 - (1, 53) train_items_per_sec : 87240.07248084983 items/s
+DLL 2023-07-12 20:15:08.574598 - (1, 53) train_iter_time : 0.7336078270000144 
+DLL 2023-07-12 20:15:08.575789 - (1, 54) glob_iter/iters_per_epoch : 132/78 
+DLL 2023-07-12 20:15:08.792031 - (1, 54) train_loss : 0.0031013810075819492 
+DLL 2023-07-12 20:15:09.310383 - (1, 54) train_items_per_sec : 87122.31040577187 items/s
+DLL 2023-07-12 20:15:09.310417 - (1, 54) train_iter_time : 0.7345994350002911 
+DLL 2023-07-12 20:15:09.311670 - (1, 55) glob_iter/iters_per_epoch : 133/78 
+DLL 2023-07-12 20:15:09.518849 - (1, 55) train_loss : 0.0044684698805212975 
+DLL 2023-07-12 20:15:10.043296 - (1, 55) train_items_per_sec : 87475.72343652259 items/s
+DLL 2023-07-12 20:15:10.043333 - (1, 55) train_iter_time : 0.7316315600000962 
+DLL 2023-07-12 20:15:10.044291 - (1, 56) glob_iter/iters_per_epoch : 134/78 
+DLL 2023-07-12 20:15:10.258149 - (1, 56) train_loss : 0.0013136632042005658 
+DLL 2023-07-12 20:15:10.776174 - (1, 56) train_items_per_sec : 87444.94309254376 items/s
+DLL 2023-07-12 20:15:10.776213 - (1, 56) train_iter_time : 0.731889092000074 
+DLL 2023-07-12 20:15:10.777831 - (1, 57) glob_iter/iters_per_epoch : 135/78 
+DLL 2023-07-12 20:15:10.992960 - (1, 57) train_loss : 0.002914617070928216 
+DLL 2023-07-12 20:15:11.512892 - (1, 57) train_items_per_sec : 87067.04590114485 items/s
+DLL 2023-07-12 20:15:11.512928 - (1, 57) train_iter_time : 0.7350657110000611 
+DLL 2023-07-12 20:15:11.514330 - (1, 58) glob_iter/iters_per_epoch : 136/78 
+DLL 2023-07-12 20:15:11.721136 - (1, 58) train_loss : 0.0010785504709929228 
+DLL 2023-07-12 20:15:12.243561 - (1, 58) train_items_per_sec : 87763.13927772563 items/s
+DLL 2023-07-12 20:15:12.243598 - (1, 58) train_iter_time : 0.7292355370000223 
+DLL 2023-07-12 20:15:12.245081 - (1, 59) glob_iter/iters_per_epoch : 137/78 
+DLL 2023-07-12 20:15:12.459580 - (1, 59) train_loss : 0.0016454551368951797 
+DLL 2023-07-12 20:15:12.978033 - (1, 59) train_items_per_sec : 87317.56182428781 items/s
+DLL 2023-07-12 20:15:12.978067 - (1, 59) train_iter_time : 0.7329567920000954 
+DLL 2023-07-12 20:15:12.979430 - (1, 60) glob_iter/iters_per_epoch : 138/78 
+DLL 2023-07-12 20:15:13.195534 - (1, 60) train_loss : 0.003099239431321621 
+DLL 2023-07-12 20:15:13.715033 - (1, 60) train_items_per_sec : 87002.85927280386 items/s
+DLL 2023-07-12 20:15:13.715067 - (1, 60) train_iter_time : 0.7356080080003267 
+DLL 2023-07-12 20:15:13.716086 - (1, 61) glob_iter/iters_per_epoch : 139/78 
+DLL 2023-07-12 20:15:13.923516 - (1, 61) train_loss : 0.002401873469352722 
+DLL 2023-07-12 20:15:14.446201 - (1, 61) train_items_per_sec : 87656.84478612008 items/s
+DLL 2023-07-12 20:15:14.446247 - (1, 61) train_iter_time : 0.7301198230002228 
+DLL 2023-07-12 20:15:14.447031 - (1, 62) glob_iter/iters_per_epoch : 140/78 
+DLL 2023-07-12 20:15:14.660622 - (1, 62) train_loss : 0.002212476683780551 
+DLL 2023-07-12 20:15:15.179120 - (1, 62) train_items_per_sec : 87420.65856410429 items/s
+DLL 2023-07-12 20:15:15.179153 - (1, 62) train_iter_time : 0.7320924029995695 
+DLL 2023-07-12 20:15:15.180517 - (1, 63) glob_iter/iters_per_epoch : 141/78 
+DLL 2023-07-12 20:15:15.396058 - (1, 63) train_loss : 0.0017173909582197666 
+DLL 2023-07-12 20:15:15.915064 - (1, 63) train_items_per_sec : 87127.93944911224 items/s
+DLL 2023-07-12 20:15:15.915133 - (1, 63) train_iter_time : 0.7345519749997038 
+DLL 2023-07-12 20:15:15.916324 - (1, 64) glob_iter/iters_per_epoch : 142/78 
+DLL 2023-07-12 20:15:16.123622 - (1, 64) train_loss : 0.0014445006381720304 
+DLL 2023-07-12 20:15:16.646535 - (1, 64) train_items_per_sec : 87645.1286756137 items/s
+DLL 2023-07-12 20:15:16.646568 - (1, 64) train_iter_time : 0.7302174229998855 
+DLL 2023-07-12 20:15:16.647708 - (1, 65) glob_iter/iters_per_epoch : 143/78 
+DLL 2023-07-12 20:15:16.861250 - (1, 65) train_loss : 0.0023423107340931892 
+DLL 2023-07-12 20:15:17.379564 - (1, 65) train_items_per_sec : 87448.14547012874 items/s
+DLL 2023-07-12 20:15:17.379602 - (1, 65) train_iter_time : 0.7318622899997536 
+DLL 2023-07-12 20:15:17.381181 - (1, 66) glob_iter/iters_per_epoch : 144/78 
+DLL 2023-07-12 20:15:17.596303 - (1, 66) train_loss : 0.0030069113709032536 
+DLL 2023-07-12 20:15:18.114314 - (1, 66) train_items_per_sec : 87295.8496021872 items/s
+DLL 2023-07-12 20:15:18.114351 - (1, 66) train_iter_time : 0.7331390929998634 
+DLL 2023-07-12 20:15:18.115688 - (1, 67) glob_iter/iters_per_epoch : 145/78 
+DLL 2023-07-12 20:15:18.321511 - (1, 67) train_loss : 0.0028555220924317837 
+DLL 2023-07-12 20:15:18.845930 - (1, 67) train_items_per_sec : 87641.58684933634 items/s
+DLL 2023-07-12 20:15:18.845971 - (1, 67) train_iter_time : 0.7302469330002168 
+DLL 2023-07-12 20:15:18.847246 - (1, 68) glob_iter/iters_per_epoch : 146/78 
+DLL 2023-07-12 20:15:19.060897 - (1, 68) train_loss : 0.002050295239314437 
+DLL 2023-07-12 20:15:19.578613 - (1, 68) train_items_per_sec : 87506.77994670586 items/s
+DLL 2023-07-12 20:15:19.578646 - (1, 68) train_iter_time : 0.7313719009998749 
+DLL 2023-07-12 20:15:19.579786 - (1, 69) glob_iter/iters_per_epoch : 147/78 
+DLL 2023-07-12 20:15:19.795521 - (1, 69) train_loss : 0.00243458547629416 
+DLL 2023-07-12 20:15:20.315167 - (1, 69) train_items_per_sec : 87029.17478207896 items/s
+DLL 2023-07-12 20:15:20.315203 - (1, 69) train_iter_time : 0.735385578000205 
+DLL 2023-07-12 20:15:20.315916 - (1, 70) glob_iter/iters_per_epoch : 148/78 
+DLL 2023-07-12 20:15:20.523467 - (1, 70) train_loss : 0.0019123477395623922 
+DLL 2023-07-12 20:15:21.046291 - (1, 70) train_items_per_sec : 87625.70076203147 items/s
+DLL 2023-07-12 20:15:21.046340 - (1, 70) train_iter_time : 0.7303793230003066 
+DLL 2023-07-12 20:15:21.047635 - (1, 71) glob_iter/iters_per_epoch : 149/78 
+DLL 2023-07-12 20:15:21.260723 - (1, 71) train_loss : 0.0019573108293116093 
+DLL 2023-07-12 20:15:21.777630 - (1, 71) train_items_per_sec : 87671.17775193813 items/s
+DLL 2023-07-12 20:15:21.777664 - (1, 71) train_iter_time : 0.7300004590001663 
+DLL 2023-07-12 20:15:21.779005 - (1, 72) glob_iter/iters_per_epoch : 150/78 
+DLL 2023-07-12 20:15:21.994701 - (1, 72) train_loss : 0.0021032809745520353 
+DLL 2023-07-12 20:15:22.513663 - (1, 72) train_items_per_sec : 87114.58903043604 items/s
+DLL 2023-07-12 20:15:22.513697 - (1, 72) train_iter_time : 0.734664545999749 
+DLL 2023-07-12 20:15:22.515048 - (1, 73) glob_iter/iters_per_epoch : 151/78 
+DLL 2023-07-12 20:15:22.721873 - (1, 73) train_loss : 0.0020965044386684895 
+DLL 2023-07-12 20:15:23.245873 - (1, 73) train_items_per_sec : 87571.58564927286 items/s
+DLL 2023-07-12 20:15:23.245908 - (1, 73) train_iter_time : 0.7308306629997787 
+DLL 2023-07-12 20:15:23.247385 - (1, 74) glob_iter/iters_per_epoch : 152/78 
+DLL 2023-07-12 20:15:23.460389 - (1, 74) train_loss : 0.002126089995726943 
+DLL 2023-07-12 20:15:23.978874 - (1, 74) train_items_per_sec : 87492.05131293563 items/s
+DLL 2023-07-12 20:15:23.978908 - (1, 74) train_iter_time : 0.7314950220002174 
+DLL 2023-07-12 20:15:23.979989 - (1, 75) glob_iter/iters_per_epoch : 153/78 
+DLL 2023-07-12 20:15:24.194976 - (1, 75) train_loss : 0.0019275132799521089 
+DLL 2023-07-12 20:15:24.713459 - (1, 75) train_items_per_sec : 87255.7330288679 items/s
+DLL 2023-07-12 20:15:24.713496 - (1, 75) train_iter_time : 0.7334761600000093 
+DLL 2023-07-12 20:15:24.714797 - (1, 76) glob_iter/iters_per_epoch : 154/78 
+DLL 2023-07-12 20:15:24.921763 - (1, 76) train_loss : 0.0025893542915582657 
+DLL 2023-07-12 20:15:25.444123 - (1, 76) train_items_per_sec : 87751.74450125775 items/s
+DLL 2023-07-12 20:15:25.444164 - (1, 76) train_iter_time : 0.7293302299999596 
+DLL 2023-07-12 20:15:25.445833 - (1, 77) glob_iter/iters_per_epoch : 155/78 
+DLL 2023-07-12 20:15:25.658706 - (1, 77) train_loss : 0.0032133397180587053 
+DLL 2023-07-12 20:15:26.179536 - (1, 77) train_items_per_sec : 87228.05782395657 items/s
+DLL 2023-07-12 20:15:26.179570 - (1, 77) train_iter_time : 0.7337088730000687 
+DLL 2023-07-12 20:15:26.200339 - (1,) train_items_per_sec : 87506.2040407612 items/s
+DLL 2023-07-12 20:15:26.200389 - (1,) train_loss : 0.0032133397180587053 
+DLL 2023-07-12 20:15:26.200401 - (1,) train_epoch_time : 57.32604694099973 
+DLL 2023-07-12 20:15:26.434751 - (1, 156, 0) val_items_per_sec : 315944.2006011066 items/s
+DLL 2023-07-12 20:15:26.646115 - (1, 156, 1) val_items_per_sec : 304601.4726168422 items/s
+DLL 2023-07-12 20:15:26.857532 - (1, 156, 2) val_items_per_sec : 303853.08885454264 items/s
+DLL 2023-07-12 20:15:27.069150 - (1, 156, 3) val_items_per_sec : 303804.9466670348 items/s
+DLL 2023-07-12 20:15:27.286081 - (1, 156, 4) val_items_per_sec : 296195.19363216305 items/s
+DLL 2023-07-12 20:15:27.503244 - (1, 156, 5) val_items_per_sec : 295856.87760817714 items/s
+DLL 2023-07-12 20:15:27.715349 - (1, 156, 6) val_items_per_sec : 302802.8052179698 items/s
+DLL 2023-07-12 20:15:27.927000 - (1, 156, 7) val_items_per_sec : 303392.70072767424 items/s
+DLL 2023-07-12 20:15:28.138516 - (1, 156, 8) val_items_per_sec : 303808.8736861196 items/s
+DLL 2023-07-12 20:15:28.355033 - (1, 156, 9) val_items_per_sec : 296799.4772137651 items/s
+DLL 2023-07-12 20:15:28.571469 - (1, 156, 10) val_items_per_sec : 296912.1195238339 items/s
+DLL 2023-07-12 20:15:28.784879 - (1, 156, 11) val_items_per_sec : 300788.26405396493 items/s
+DLL 2023-07-12 20:15:28.895389 - (1, 156, 12) val_items_per_sec : 291827.0275184809 items/s
+DLL 2023-07-12 20:15:28.911112 - (1,) val_loss : 0.0025272821971716788 None
+DLL 2023-07-12 20:15:28.911155 - (1,) val_items_per_sec : 301275.92676320573 items/s
+DLL 2023-07-12 20:15:28.911967 - () run_time : 128.83509841500018 s
+DLL 2023-07-12 20:15:28.911989 - () val_loss : 0.0025272821971716788 None
+DLL 2023-07-12 20:15:28.912000 - () train_loss : 0.0032133397180587053 
+DLL 2023-07-12 20:15:28.912009 - () train_items_per_sec : 87506.2040407612 items/s
+DLL 2023-07-12 20:15:28.912017 - () val_items_per_sec : 301275.92676320573 items/s
+DONE!
diff --git a/pytorch/results/4090laptop_v1/PyTorch_waveglow_FP32/benchmark.para b/pytorch/results/4090laptop_v1/PyTorch_waveglow_FP32/benchmark.para
new file mode 100755
index 00000000..6ea5ca92
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/PyTorch_waveglow_FP32/benchmark.para
@@ -0,0 +1,2 @@
+GLOBAL_BATCH 8
+GPU 1
diff --git a/pytorch/results/4090laptop_v1/summary.txt b/pytorch/results/4090laptop_v1/summary.txt
new file mode 100644
index 00000000..ac09849c
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/summary.txt
@@ -0,0 +1,22 @@
+Check results folder : /results/4090laptop_v1
+['PyTorch_SSD_AMP', 'PyTorch_SSD_FP32', 'PyTorch_bert_base_squad_FP16', 'PyTorch_bert_base_squad_FP32', 'PyTorch_bert_large_squad_FP16', 'PyTorch_bert_large_squad_FP32', 'PyTorch_gnmt_FP16', 'PyTorch_gnmt_FP32', 'PyTorch_ncf_FP16', 'PyTorch_ncf_FP32', 'PyTorch_resnet50_AMP', 'PyTorch_resnet50_FP32', 'PyTorch_tacotron2_FP16', 'PyTorch_tacotron2_FP32', 'PyTorch_transformerxlbase_FP16', 'PyTorch_transformerxlbase_FP32', 'PyTorch_transformerxllarge_FP16', 'PyTorch_transformerxllarge_FP32', 'PyTorch_waveglow_FP16', 'PyTorch_waveglow_FP32', 'summary.txt', 'sys_pytorch.txt']
+PyTorch_SSD_AMP                     :  sucessful
+PyTorch_SSD_FP32                    :  sucessful
+PyTorch_bert_base_squad_FP16        :  sucessful
+PyTorch_bert_base_squad_FP32        :  sucessful
+PyTorch_bert_large_squad_FP16       :  sucessful
+PyTorch_bert_large_squad_FP32       :  sucessful
+PyTorch_gnmt_FP16                   :  sucessful
+PyTorch_gnmt_FP32                   :  sucessful
+PyTorch_ncf_FP16                    :  sucessful
+PyTorch_ncf_FP32                    :  sucessful
+PyTorch_resnet50_AMP                :  sucessful
+PyTorch_resnet50_FP32               :  sucessful
+PyTorch_tacotron2_FP16              :  sucessful
+PyTorch_tacotron2_FP32              :  sucessful
+PyTorch_transformerxlbase_FP16      :  sucessful
+PyTorch_transformerxlbase_FP32      :  sucessful
+PyTorch_transformerxllarge_FP16     :  sucessful
+PyTorch_transformerxllarge_FP32     :  sucessful
+PyTorch_waveglow_FP16               :  sucessful
+PyTorch_waveglow_FP32               :  sucessful
diff --git a/pytorch/results/4090laptop_v1/sys_pytorch.txt b/pytorch/results/4090laptop_v1/sys_pytorch.txt
new file mode 100755
index 00000000..84ccbc76
--- /dev/null
+++ b/pytorch/results/4090laptop_v1/sys_pytorch.txt
@@ -0,0 +1,10 @@
+CPU: 13th Gen Intel(R) Core(TM) i9-13950HX
+CPU Memory: 30Gi
+GPU: NVIDIA_GeForce_RTX_4090_Laptop_GPU
+GPU Memory: 16376_MiB
+NVIDIA driver: 525.116.04
+CUDA Version: V11.8.89
+CUDNN Version: ..
+Motherboard: Razer SO690 3
+OS: Ubuntu 20.04.5 LTS
+PyTorch Version: 1.13.0a0+d0d6b1f
diff --git a/pytorch/scripts/config_v1/config_pytorch_4090laptop_v1.sh b/pytorch/scripts/config_v1/config_pytorch_4090laptop_v1.sh
new file mode 100644
index 00000000..e51f9172
--- /dev/null
+++ b/pytorch/scripts/config_v1/config_pytorch_4090laptop_v1.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+source config_v1/config_pytorch_16GB.sh
+
+declare -A BATCH_SIZE_FIX=(
+)
+source config_v1/fix.sh